sidekiq-unique-jobs 6.0.7 → 7.1.33

data/lib/sidekiq_unique_jobs/locksmith.rb

@@ -3,162 +3,391 @@
 module SidekiqUniqueJobs
   # Lock manager class that handles all the various locks
   #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
-  class Locksmith
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  class Locksmith # rubocop:disable Metrics/ClassLength
+    # includes "SidekiqUniqueJobs::Connection"
+    # @!parse include SidekiqUniqueJobs::Connection
     include SidekiqUniqueJobs::Connection
 
+    # includes "SidekiqUniqueJobs::Logging"
+    # @!parse include SidekiqUniqueJobs::Logging
+    include SidekiqUniqueJobs::Logging
+
+    # includes "SidekiqUniqueJobs::Reflectable"
+    # @!parse include SidekiqUniqueJobs::Reflectable
+    include SidekiqUniqueJobs::Reflectable
+
+    # includes "SidekiqUniqueJobs::Timing"
+    # @!parse include SidekiqUniqueJobs::Timing
+    include SidekiqUniqueJobs::Timing
+
+    # includes "SidekiqUniqueJobs::Script::Caller"
+    # @!parse include SidekiqUniqueJobs::Script::Caller
+    include SidekiqUniqueJobs::Script::Caller
+
+    # includes "SidekiqUniqueJobs::JSON"
+    # @!parse include SidekiqUniqueJobs::JSON
+    include SidekiqUniqueJobs::JSON
+
+    #
+    # @return [Float] used to take into consideration the inaccuracy of redis timestamps
+    CLOCK_DRIFT_FACTOR = 0.01
+    NETWORK_FACTOR = 0.04
+
+    #
+    # @!attribute [r] key
+    #   @return [Key] the key used for locking
+    attr_reader :key
+    #
+    # @!attribute [r] job_id
+    #   @return [String] a sidekiq JID
+    attr_reader :job_id
+    #
+    # @!attribute [r] config
+    #   @return [LockConfig] the configuration for this lock
+    attr_reader :config
+    #
+    # @!attribute [r] item
+    #   @return [Hash] a sidekiq job hash
+    attr_reader :item
+
+    #
+    # Initialize a new Locksmith instance
+    #
     # @param [Hash] item a Sidekiq job hash
-    # @option item [Integer] :lock_expiration the configured expiration
+    # @option item [Integer] :lock_ttl the configured expiration
     # @option item [String] :jid the sidekiq job id
-    # @option item [String] :unique_digest the unique digest (See: {UniqueArgs#unique_digest})
+    # @option item [String] :unique_digest the unique digest (See: {LockDigest#lock_digest})
     # @param [Sidekiq::RedisConnection, ConnectionPool] redis_pool the redis connection
+    #
     def initialize(item, redis_pool = nil)
-      @concurrency   = 1 # removed in a0cff5bc42edbe7190d6ede7e7f845074d2d7af6
-      @expiration    = item[LOCK_EXPIRATION_KEY]
-      @jid           = item[JID_KEY]
-      @unique_digest = item[UNIQUE_DIGEST_KEY]
-      @redis_pool    = redis_pool
+      @item        = item
+      @key         = Key.new(item[LOCK_DIGEST] || item[UNIQUE_DIGEST]) # fallback until can be removed
+      @job_id      = item[JID]
+      @config      = LockConfig.new(item)
+      @redis_pool  = redis_pool
     end
 
-    # Checks if the exists key is created in redis
-    # @return [true, false]
-    def exists?
-      redis(redis_pool) { |conn| conn.exists(exists_key) }
-    end
-
-    # The number of available resourced for this lock
-    # @return [Integer] the number of available resources
-    def available_count
-      return concurrency unless exists?
-
-      redis(redis_pool) { |conn| conn.llen(available_key) }
-    end
-
-    # Deletes the lock unless it has an expiration set
+    #
+    # Deletes the lock unless it has a pttl set
+    #
+    #
     def delete
-      return if expiration
+      return if config.pttl.positive?
 
       delete!
     end
 
-    # Deletes the lock regardless of if it has an expiration set
+    #
+    # Deletes the lock regardless of if it has a pttl set
+    #
     def delete!
-      Scripts.call(
-        :delete,
-        redis_pool,
-        keys: [exists_key, grabbed_key, available_key, version_key, UNIQUE_SET, unique_digest],
-      )
+      call_script(:delete, key.to_a, [job_id, config.pttl, config.type, config.limit]).to_i.positive?
+    end
+
+    #
+    # Create a lock for the Sidekiq job
+    #
+    # @return [String] the Sidekiq job_id that was locked/queued
+    #
+    def lock(wait: nil)
+      method_name = wait ? :primed_async : :primed_sync
+      redis(redis_pool) do |conn|
+        lock!(conn, method(method_name), wait) do
+          return job_id
+        end
+      end
     end
 
-    # Create a lock for the item
-    # @param [Integer] timeout the number of seconds to wait for a lock.
-    #   nil means wait indefinitely
-    # @yield the block to execute if a lock is successful
-    # @return the Sidekiq job_id (jid)
-    def lock(timeout = nil, &block)
-      Scripts.call(:lock, redis_pool,
-                   keys: [exists_key, grabbed_key, available_key, UNIQUE_SET, unique_digest],
-                   argv: [jid, expiration])
+    def execute(&block)
+      raise SidekiqUniqueJobs::InvalidArgument, "#execute needs a block" unless block
 
-      grab_token(timeout) do |token|
-        touch_grabbed_token(token)
-        return_token_or_block_value(token, &block)
+      redis(redis_pool) do |conn|
+        lock!(conn, method(:primed_async), &block)
       end
     end
-    alias wait lock
 
+    #
     # Removes the lock keys from Redis if locked by the provided jid/token
+    #
     # @return [false] unless locked?
     # @return [String] Sidekiq job_id (jid) if successful
-    def unlock(token = nil)
-      token ||= jid
-      return false unless locked?(token)
+    #
+    def unlock(conn = nil)
+      return false unless locked?(conn)
 
-      unlock!(token)
+      unlock!(conn)
     end
 
+    #
     # Removes the lock keys from Redis
+    #
     # @return [false] unless locked?
     # @return [String] Sidekiq job_id (jid) if successful
-    def unlock!(token = nil)
-      token ||= jid
-
-      Scripts.call(
-        :unlock,
-        redis_pool,
-        keys: [exists_key, grabbed_key, available_key, version_key, UNIQUE_SET, unique_digest],
-        argv: [token, expiration],
-      )
+    #
+    def unlock!(conn = nil)
+      call_script(:unlock, key.to_a, argv, conn) do |unlocked_jid|
+        if unlocked_jid == job_id
+          reflect(:debug, :unlocked, item, unlocked_jid)
+          reflect(:unlocked, item)
+        end
+
+        unlocked_jid
+      end
     end
 
     # Checks if this instance is considered locked
-    # @param [String] token the unique token to check for a lock.
-    #   nil will default to the jid provided in the initializer
+    #
+    # @param [Sidekiq::RedisConnection, ConnectionPool] conn the redis connection
+    #
+    # @return [true, false] true when the :LOCKED hash contains the job_id
+    #
+    def locked?(conn = nil)
+      return taken?(conn) if conn
+
+      redis { |rcon| taken?(rcon) }
+    end
+
+    #
+    # Nicely formatted string with information about self
+    #
+    #
+    # @return [String]
+    #
+    def to_s
+      "Locksmith##{object_id}(digest=#{key} job_id=#{job_id} locked=#{locked?})"
+    end
+
+    #
+    # @see to_s
+    #
+    def inspect
+      to_s
+    end
+
+    #
+    # Compare this locksmith with another
+    #
+    # @param [Locksmith] other the locksmith to compare with
+    #
     # @return [true, false]
-    def locked?(token = nil)
-      token ||= jid
-      redis(redis_pool) { |conn| conn.hexists(grabbed_key, token) }
+    #
+    def ==(other)
+      key == other.key && job_id == other.job_id
     end
 
     private
 
-    attr_reader :concurrency, :unique_digest, :expiration, :jid, :redis_pool
+    attr_reader :redis_pool
 
-    def grab_token(timeout = nil)
-      redis(redis_pool) do |conn|
-        if timeout.nil? || timeout.positive?
-          # passing timeout 0 to blpop causes it to block
-          _key, token = conn.blpop(available_key, timeout || 0)
-        else
-          token = conn.lpop(available_key)
+    #
+    # Used to reduce some duplication from the two methods
+    #
+    # @see lock
+    # @see execute
+    #
+    # @param [Sidekiq::RedisConnection, ConnectionPool] conn the redis connection
+    # @param [Method] primed_method reference to the method to use for getting a primed token
+    # @param [nil, Integer, Float] time to wait before timeout
+    #
+    # @yieldparam [string] job_id the sidekiq JID
+    # @yieldreturn [void] whatever the calling block returns
+    def lock!(conn, primed_method, wait = nil)
+      return yield if locked?(conn)
+
+      enqueue(conn) do |queued_jid|
+        reflect(:debug, :queued, item, queued_jid)
+
+        primed_method.call(conn, wait) do |primed_jid|
+          reflect(:debug, :primed, item, primed_jid)
+          locked_jid = call_script(:lock, key.to_a, argv, conn)
+
+          if locked_jid
+            reflect(:debug, :locked, item, locked_jid)
+            return yield
+          end
         end
+      end
+    end
 
-        return yield jid if token
+    #
+    # Prepares all the various lock data
+    #
+    # @param [Redis] conn a redis connection
+    #
+    # @return [nil] when redis was already prepared for this lock
+    # @return [yield<String>] when successfully enqueued
+    #
+    def enqueue(conn)
+      queued_jid, elapsed = timed do
+        call_script(:queue, key.to_a, argv, conn)
       end
+
+      return unless queued_jid
+      return unless [job_id, "1"].include?(queued_jid)
+
+      validity = config.pttl - elapsed - drift(config.pttl)
+      return unless validity >= 0 || config.pttl.zero?
+
+      write_lock_info(conn)
+      yield job_id
+    end
+
+    #
+    # Pops an enqueued token
+    # @note Used for runtime locks to avoid problems with blocking commands
+    #   in current thread
+    #
+    # @param [Redis] conn a redis connection
+    #
+    # @return [nil] when lock was not possible
+    # @return [Object] whatever the block returns when lock was acquired
+    #
+    def primed_async(conn, wait = nil, &block)
+      timeout = (wait || config.timeout).to_i
+      timeout = 1 if timeout.zero?
+
+      brpoplpush_timeout = timeout
+      concurrent_timeout = add_drift(timeout)
+
+      reflect(:debug, :timeouts, item,
+              timeouts: {
+                brpoplpush_timeout: brpoplpush_timeout,
+                concurrent_timeout: concurrent_timeout,
+              })
+
+      # NOTE: When debugging, change .value to .value!
+      primed_jid = Concurrent::Promises
+                   .future(conn) { |red_con| pop_queued(red_con, timeout) }
+                   .value
+
+      handle_primed(primed_jid, &block)
+    end
+
+    #
+    # Pops an enqueued token
+    # @note Used for non-runtime locks
+    #
+    # @param [Redis] conn a redis connection
+    #
+    # @return [nil] when lock was not possible
+    # @return [Object] whatever the block returns when lock was acquired
+    #
+    def primed_sync(conn, wait = nil, &block)
+      primed_jid = pop_queued(conn, wait)
+      handle_primed(primed_jid, &block)
     end
 
-    def touch_grabbed_token(token)
-      redis(redis_pool) { |conn| conn.hset(grabbed_key, token, current_time.to_f) }
+    def handle_primed(primed_jid)
+      return yield job_id if [job_id, "1"].include?(primed_jid)
+
+      reflect(:timeout, item) unless config.wait_for_lock?
     end
 
-    def return_token_or_block_value(token)
-      return token unless block_given?
+    #
+    # Does the actual popping of the enqueued token
+    #
+    # @param [Redis] conn a redis connection
+    #
+    # @return [String] a previously enqueued token (now taken off the queue)
+    #
+    def pop_queued(conn, wait = 1)
+      wait ||= config.timeout if config.wait_for_lock?
 
-      # The reason for begin is to only signal when we have a block
-      begin
-        yield token
-      ensure
-        unlock(token)
+      if wait.nil?
+        rpoplpush(conn)
+      else
+        brpoplpush(conn, wait)
       end
     end
 
-    def available_key
-      @available_key ||= namespaced_key('AVAILABLE')
+    #
+    # @api private
+    #
+    def brpoplpush(conn, wait)
+      # passing timeout 0 to brpoplpush causes it to block indefinitely
+      raise InvalidArgument, "wait must be an integer" unless wait.is_a?(Integer)
+
+      if defined?(::Redis::Namespace) && conn.instance_of?(::Redis::Namespace)
+        return conn.brpoplpush(key.queued, key.primed, wait)
+      end
+
+      if VersionCheck.satisfied?(redis_version, ">= 6.2.0") && conn.respond_to?(:blmove)
+        conn.blmove(key.queued, key.primed, "RIGHT", "LEFT", timeout: wait)
+      else
+        conn.brpoplpush(key.queued, key.primed, timeout: wait)
+      end
     end
 
-    def exists_key
-      @exists_key ||= namespaced_key('EXISTS')
+    #
+    # @api private
+    #
+    def rpoplpush(conn)
+      conn.rpoplpush(key.queued, key.primed)
     end
 
-    def grabbed_key
-      @grabbed_key ||= namespaced_key('GRABBED')
+    #
+    # Writes lock information to redis.
+    #   The lock information contains information about worker, queue, limit etc.
+    #
+    #
+    # @return [void]
+    #
+    def write_lock_info(conn)
+      return unless config.lock_info?
+
+      conn.set(key.info, lock_info)
+    end
+
+    #
+    # Used to combat redis imprecision with ttl/pttl
+    #
+    # @param [Integer] val the value to compute drift for
+    #
+    # @return [Integer] a computed drift value
+    #
+    def drift(val)
+      # Add 2 milliseconds to the drift to account for Redis expires
+      # precision, which is 1 millisecond, plus 1 millisecond min drift
+      # for small TTLs.
+      (val + 2).to_f * CLOCK_DRIFT_FACTOR
     end
 
-    def version_key
-      @version_key ||= namespaced_key('VERSION')
+    def add_drift(val)
+      val = val.to_f
+      val + drift(val)
     end
 
-    def namespaced_key(variable)
-      "#{unique_digest}:#{variable}"
+    #
+    # Checks if the lock has been taken
+    #
+    # @param [Redis] conn a redis connection
+    #
+    # @return [true, false]
+    #
+    def taken?(conn)
+      conn.hexists(key.locked, job_id)
     end
 
-    def current_time
-      seconds, microseconds_with_frac = redis_time
-      Time.at(seconds, microseconds_with_frac)
+    def argv
+      [job_id, config.pttl, config.type, config.limit]
+    end
+
+    def lock_info
+      @lock_info ||= dump_json(
+        WORKER => item[CLASS],
+        QUEUE => item[QUEUE],
+        LIMIT => item[LOCK_LIMIT],
+        TIMEOUT => item[LOCK_TIMEOUT],
+        TTL => item[LOCK_TTL],
+        TYPE => config.type,
+        LOCK_ARGS => item[LOCK_ARGS],
+        TIME => now_f,
+      )
     end
 
-    def redis_time
-      redis(&:time)
+    def redis_version
+      @redis_version ||= SidekiqUniqueJobs.config.redis_version
     end
   end
 end

data/lib/sidekiq_unique_jobs/logging/middleware_context.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Provides the sidekiq middleware that makes the gem work
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  module Logging
+    #
+    # Context aware logging for Sidekiq Middlewares
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
+    module Middleware
+      include Logging
+
+      def self.included(base)
+        base.class_eval do
+          extend Logging::Middleware
+        end
+      end
+
+      #
+      # Provides a logging context for Sidekiq Middlewares
+      #
+      #
+      # @return [Hash] when logger responds to `:with_context`
+      # @return [String] when logger does not responds to `:with_context`
+      #
+      def logging_context
+        middleware = is_a?(SidekiqUniqueJobs::Middleware::Client) ? :client : :server
+        digest     = item[LOCK_DIGEST]
+        lock_type  = item[LOCK]
+
+        if logger_context_hash?
+          { "uniquejobs" => middleware, lock_type => digest }
+        else
+          "uniquejobs-#{middleware} #{"DIG-#{digest}" if digest}"
+        end
+      end
+    end
+  end
+end