sidekiq-unique-jobs 6.0.24 → 7.0.4

data/lib/sidekiq_unique_jobs/lock_timeout.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  # Calculates timeout and expiration
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  class LockTimeout
+    # includes "SidekiqUniqueJobs::SidekiqWorkerMethods"
+    # @!parse include SidekiqUniqueJobs::SidekiqWorkerMethods
+    include SidekiqUniqueJobs::SidekiqWorkerMethods
+
+    #
+    # Calculates the timeout for a Sidekiq job
+    #
+    # @param [Hash] item sidekiq job hash
+    #
+    # @return [Integer] timeout in seconds
+    #
+    def self.calculate(item)
+      new(item).calculate
+    end
+
+    # @!attribute [r] item
+    #   @return [Hash] the Sidekiq job hash
+    attr_reader :item
+
+    # @param [Hash] item the Sidekiq job hash
+    # @option item [Integer, nil] :lock_ttl the configured lock expiration
+    # @option item [Integer, nil] :lock_timeout the configured lock timeout
+    # @option item [String] :class the class of the sidekiq worker
+    # @option item [Float] :at the unix time the job is scheduled at
+    def initialize(item)
+      @item         = item
+      @worker_class = item[CLASS]
+    end
+
+    #
+    # Finds a lock timeout in either of
+    #  default worker options, {default_lock_timeout} or provided worker_options
+    #
+    #
+    # @return [Integer, nil]
+    #
+    def calculate
+      timeout = default_worker_options[LOCK_TIMEOUT]
+      timeout = default_lock_timeout if default_lock_timeout
+      timeout = worker_options[LOCK_TIMEOUT] if worker_options.key?(LOCK_TIMEOUT)
+      timeout
+    end
+
+    #
+    # The configured default_lock_timeout
+    # @see SidekiqUniqueJobs::Config#lock_timeout
+    #
+    #
+    # @return [Integer, nil]
+    #
+    def default_lock_timeout
+      SidekiqUniqueJobs.config.lock_timeout
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lock_ttl.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  # Calculates timeout and expiration
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  class LockTTL
+    # includes "SidekiqUniqueJobs::SidekiqWorkerMethods"
+    # @!parse include SidekiqUniqueJobs::SidekiqWorkerMethods
+    include SidekiqUniqueJobs::SidekiqWorkerMethods
+
+    #
+    # Computes lock ttl from job arguments, sidekiq_options.
+    #   Falls back to {SidekiqUniqueJobs::Config#lock_ttl}
+    #
+    # @note this method takes into consideration the time
+    #   until a job is scheduled
+    #
+    #
+    # @return [Integer] the number of seconds to live
+    #
+    def self.calculate(item)
+      new(item).calculate
+    end
+
+    # @!attribute [r] item
+    #   @return [Hash] the Sidekiq job hash
+    attr_reader :item
+
+    # @param [Hash] item the Sidekiq job hash
+    # @option item [Integer, nil] :lock_ttl the configured lock expiration
+    # @option item [Integer, nil] :lock_timeout the configured lock timeout
+    # @option item [String] :class the class of the sidekiq worker
+    # @option item [Float] :at the unix time the job is scheduled at
+    def initialize(item)
+      @item         = item
+      @worker_class = item[CLASS]
+    end
+
+    #
+    # Calculates the time until the job is scheduled starting from now
+    #
+    #
+    # @return [Integer] the number of seconds until job is scheduled
+    #
+    def time_until_scheduled
+      return 0 unless scheduled_at
+
+      scheduled_at.to_i - Time.now.utc.to_i
+    end
+
+    # The time a job is scheduled
+    # @return [Float] the exact unix time the job is scheduled at
+    def scheduled_at
+      @scheduled_at ||= item[AT]
+    end
+
+    #
+    # Computes lock ttl from job arguments, sidekiq_options.
+    #   Falls back to {SidekiqUniqueJobs::Config#lock_ttl}
+    #
+    # @note this method takes into consideration the time
+    #   until a job is scheduled
+    #
+    #
+    # @return [Integer] the number of seconds to live
+    #
+    def calculate
+      ttl = item[LOCK_TTL]
+      ttl ||= worker_options[LOCK_TTL]
+      ttl ||= item[LOCK_EXPIRATION] # TODO: Deprecate at some point
+      ttl ||= worker_options[LOCK_EXPIRATION] # TODO: Deprecate at some point
+      ttl ||= SidekiqUniqueJobs.config.lock_ttl
+      ttl && ttl.to_i + time_until_scheduled
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/locksmith.rb

@@ -3,64 +3,97 @@
 module SidekiqUniqueJobs
   # Lock manager class that handles all the various locks
   #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
-  # rubocop:disable Metrics/ClassLength
-  class Locksmith
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  class Locksmith # rubocop:disable Metrics/ClassLength
+    # includes "SidekiqUniqueJobs::Connection"
+    # @!parse include SidekiqUniqueJobs::Connection
     include SidekiqUniqueJobs::Connection
 
+    # includes "SidekiqUniqueJobs::Logging"
+    # @!parse include SidekiqUniqueJobs::Logging
+    include SidekiqUniqueJobs::Logging
+
+    # includes "SidekiqUniqueJobs::Timing"
+    # @!parse include SidekiqUniqueJobs::Timing
+    include SidekiqUniqueJobs::Timing
+
+    # includes "SidekiqUniqueJobs::Script::Caller"
+    # @!parse include SidekiqUniqueJobs::Script::Caller
+    include SidekiqUniqueJobs::Script::Caller
+
+    # includes "SidekiqUniqueJobs::JSON"
+    # @!parse include SidekiqUniqueJobs::JSON
+    include SidekiqUniqueJobs::JSON
+
+    #
+    # @return [Float] used to take into consideration the inaccuracy of redis timestamps
+    CLOCK_DRIFT_FACTOR = 0.01
+
+    #
+    # @!attribute [r] key
+    #   @return [Key] the key used for locking
+    attr_reader :key
+    #
+    # @!attribute [r] job_id
+    #   @return [String] a sidekiq JID
+    attr_reader :job_id
+    #
+    # @!attribute [r] config
+    #   @return [LockConfig] the configuration for this lock
+    attr_reader :config
+    #
+    # @!attribute [r] item
+    #   @return [Hash] a sidekiq job hash
+    attr_reader :item
+
+    #
+    # Initialize a new Locksmith instance
+    #
     # @param [Hash] item a Sidekiq job hash
-    # @option item [Integer] :lock_expiration the configured expiration
+    # @option item [Integer] :lock_ttl the configured expiration
     # @option item [String] :jid the sidekiq job id
-    # @option item [String] :unique_digest the unique digest (See: {UniqueArgs#unique_digest})
+    # @option item [String] :unique_digest the unique digest (See: {LockDigest#lock_digest})
     # @param [Sidekiq::RedisConnection, ConnectionPool] redis_pool the redis connection
+    #
     def initialize(item, redis_pool = nil)
-      # @concurrency   = 1 # removed in a0cff5bc42edbe7190d6ede7e7f845074d2d7af6
-      @ttl           = item[LOCK_EXPIRATION_KEY] || item[LOCK_TTL_KEY]
-      @jid           = item[JID_KEY]
-      @unique_digest = item[UNIQUE_DIGEST_KEY] || item[LOCK_DIGEST_KEY]
-      @lock_type     = item[LOCK_KEY] || item[UNIQUE_KEY]
-      @lock_type   &&= @lock_type.to_sym
-      @redis_pool    = redis_pool
+      @item        = item
+      @key         = Key.new(item[LOCK_DIGEST] || item[UNIQUE_DIGEST]) # fallback until can be removed
+      @job_id      = item[JID]
+      @config      = LockConfig.new(item)
+      @redis_pool  = redis_pool
     end
 
     #
-    # Deletes the lock unless it has a ttl set
+    # Deletes the lock unless it has a pttl set
     #
     #
     def delete
-      return if ttl
+      return if config.pttl.positive?
 
       delete!
     end
 
-    # Deletes the lock regardless of if it has a ttl set
+    #
+    # Deletes the lock regardless of if it has a pttl set
+    #
     def delete!
-      Scripts.call(
-        :delete,
-        redis_pool,
-        keys: [exists_key, grabbed_key, available_key, version_key, UNIQUE_SET, unique_digest],
-      )
+      call_script(:delete, key.to_a, [job_id, config.pttl, config.type, config.limit]).positive?
     end
 
     #
-    # Create a lock for the item
+    # Create a lock for the Sidekiq job
     #
-    # @param [Integer] timeout the number of seconds to wait for a lock.
+    # @return [String] the Sidekiq job_id that was locked/queued
     #
-    # @return [String] the Sidekiq job_id (jid)
-    #
-    #
-    def lock(timeout = nil, &block)
-      Scripts.call(:lock, redis_pool,
-                   keys: [exists_key, grabbed_key, available_key, UNIQUE_SET, unique_digest],
-                   argv: [jid, ttl, lock_type])
+    def lock(&block)
+      redis(redis_pool) do |conn|
+        return lock_async(conn, &block) if block
 
-      grab_token(timeout) do |token|
-        touch_grabbed_token(token)
-        return_token_or_block_value(token, &block)
+        lock_sync(conn) do
+          return job_id
+        end
       end
     end
-    alias wait lock
 
     #
     # Removes the lock keys from Redis if locked by the provided jid/token
@@ -68,122 +101,249 @@ module SidekiqUniqueJobs
     # @return [false] unless locked?
     # @return [String] Sidekiq job_id (jid) if successful
     #
-    def unlock(token = nil)
-      token ||= jid
-      return false unless locked?(token)
+    def unlock(conn = nil)
+      return false unless locked?(conn)
 
-      unlock!(token)
+      unlock!(conn)
     end
 
     #
     # Removes the lock keys from Redis
     #
-    # @param [String] token the token to unlock (defaults to jid)
-    #
     # @return [false] unless locked?
     # @return [String] Sidekiq job_id (jid) if successful
     #
-    def unlock!(token = nil)
-      token ||= jid
+    def unlock!(conn = nil)
+      call_script(:unlock, key.to_a, argv, conn)
+    end
 
-      Scripts.call(
-        :unlock,
-        redis_pool,
-        keys: [exists_key, grabbed_key, available_key, version_key, UNIQUE_SET, unique_digest],
-        argv: [token, ttl, lock_type],
-      )
+    # Checks if this instance is considered locked
+    #
+    # @return [true, false] true when the :LOCKED hash contains the job_id
+    #
+    def locked?(conn = nil)
+      return taken?(conn) if conn
+
+      redis { |rcon| taken?(rcon) }
     end
 
     #
-    # @param [String] token the unique token to check for a lock.
-    #   nil will default to the jid provided in the initializer
-    # @return [true, false]
+    # Nicely formatted string with information about self
     #
-    # Checks if this instance is considered locked
     #
-    # @param [<type>] token <description>
+    # @return [String]
+    #
+    def to_s
+      "Locksmith##{object_id}(digest=#{key} job_id=#{job_id}, locked=#{locked?})"
+    end
+
     #
-    # @return [<type>] <description>
+    # @see to_s
     #
-    def locked?(token = nil)
-      token ||= jid
+    def inspect
+      to_s
+    end
 
-      convert_legacy_lock(token)
-      redis(redis_pool) { |conn| conn.hexists(grabbed_key, token) }
+    #
+    # Compare this locksmith with another
+    #
+    # @param [Locksmith] other the locksmith to compare with
+    #
+    # @return [true, false]
+    #
+    def ==(other)
+      key == other.key && job_id == other.job_id
     end
 
     private
 
-    attr_reader :unique_digest, :ttl, :jid, :redis_pool, :lock_type
+    attr_reader :redis_pool
 
-    def convert_legacy_lock(token)
-      Scripts.call(
-        :convert_legacy_lock,
-        redis_pool,
-        keys: [grabbed_key, unique_digest],
-        argv: [token, current_time.to_f],
-      )
+    def argv
+      [job_id, config.pttl, config.type, config.limit]
     end
 
-    def grab_token(timeout = nil)
-      redis(redis_pool) do |conn|
-        if timeout.nil? || timeout.positive?
-          # passing timeout 0 to blpop causes it to block
-          _key, token = conn.blpop(available_key, timeout || 0)
-        else
-          token = conn.lpop(available_key)
-        end
+    #
+    # Used for runtime locks that need automatic unlock after yielding
+    #
+    # @param [Redis] conn a redis connection
+    #
+    # @return [nil] when lock was not possible
+    # @return [Object] whatever the block returns when lock was acquired
+    #
+    # @yieldparam [String] job_id a Sidekiq JID
+    #
+    def lock_async(conn)
+      return yield job_id if locked?(conn)
 
-        return yield jid if token
+      enqueue(conn) do
+        primed_async(conn) do
+          locked_token = call_script(:lock, key.to_a, argv, conn)
+          return yield if locked_token == job_id
+        end
       end
+    ensure
+      unlock!(conn)
     end
 
-    def touch_grabbed_token(token)
-      redis(redis_pool) do |conn|
-        conn.hset(grabbed_key, token, current_time.to_f)
-        conn.expire(grabbed_key, ttl) if ttl && lock_type == :until_expired
+    #
+    # Pops an enqueued token
+    # @note Used for runtime locks to avoid problems with blocking commands
+    #   in current thread
+    #
+    # @param [Redis] conn a redis connection
+    #
+    # @return [nil] when lock was not possible
+    # @return [Object] whatever the block returns when lock was acquired
+    #
+    def primed_async(conn)
+      return yield if Concurrent::Promises
+                      .future(conn) { |red_con| pop_queued(red_con) }
+                      .value(drift(config.ttl))
+
+      warn_about_timeout
+    end
+
+    #
+    # Used for non-runtime locks (no block was given)
+    #
+    # @param [Redis] conn a redis connection
+    #
+    # @return [nil] when lock was not possible
+    # @return [Object] whatever the block returns when lock was acquired
+    #
+    # @yieldparam [String] job_id a Sidekiq JID
+    #
+    def lock_sync(conn)
+      return yield if locked?(conn)
+
+      enqueue(conn) do
+        primed_sync(conn) do
+          locked_token = call_script(:lock, key.to_a, argv, conn)
+          return yield if locked_token
+        end
       end
     end
 
-    def return_token_or_block_value(token)
-      return token unless block_given?
+    #
+    # Pops an enqueued token
+    # @note Used for non-runtime locks
+    #
+    # @param [Redis] conn a redis connection
+    #
+    # @return [nil] when lock was not possible
+    # @return [Object] whatever the block returns when lock was acquired
+    #
+    def primed_sync(conn)
+      return yield if pop_queued(conn)
+
+      warn_about_timeout
+    end
 
-      # The reason for begin is to only signal when we have a block
-      begin
-        yield token
-      ensure
-        unlock(token)
+    #
+    # Does the actual popping of the enqueued token
+    #
+    # @param [Redis] conn a redis connection
+    #
+    # @return [String] a previously enqueued token (now taken off the queue)
+    #
+    def pop_queued(conn)
+      if config.wait_for_lock?
+        brpoplpush(conn)
+      else
+        rpoplpush(conn)
       end
     end
 
-    def available_key
-      @available_key ||= namespaced_key("AVAILABLE")
+    #
+    # @api private
+    #
+    def brpoplpush(conn)
+      # passing timeout 0 to brpoplpush causes it to block indefinitely
+      conn.brpoplpush(key.queued, key.primed, timeout: config.timeout || 0)
     end
 
-    def exists_key
-      @exists_key ||= namespaced_key("EXISTS")
+    #
+    # @api private
+    #
+    def rpoplpush(conn)
+      conn.rpoplpush(key.queued, key.primed)
     end
 
-    def grabbed_key
-      @grabbed_key ||= namespaced_key("GRABBED")
+    #
+    # Prepares all the various lock data
+    #
+    # @param [Redis] conn a redis connection
+    #
+    # @return [nil] when redis was already prepared for this lock
+    # @return [yield<String>] when successfully enqueued
+    #
+    def enqueue(conn)
+      queued_token, elapsed = timed do
+        call_script(:queue, key.to_a, argv, conn)
+      end
+
+      validity = config.pttl - elapsed - drift(config.pttl)
+
+      return unless queued_token && (validity >= 0 || config.pttl.zero?)
+
+      write_lock_info(conn)
+      yield
     end
 
-    def version_key
-      @version_key ||= namespaced_key("VERSION")
+    #
+    # Writes lock information to redis.
+    #   The lock information contains information about worker, queue, limit etc.
+    #
+    #
+    # @return [void]
+    #
+    def write_lock_info(conn)
+      return unless config.lock_info
+
+      conn.set(key.info, lock_info)
     end
 
-    def namespaced_key(variable)
-      "#{unique_digest}:#{variable}"
+    #
+    # Used to combat redis imprecision with ttl/pttl
+    #
+    # @param [Integer] val the value to compute drift for
+    #
+    # @return [Integer] a computed drift value
+    #
+    def drift(val)
+      # Add 2 milliseconds to the drift to account for Redis expires
+      # precision, which is 1 millisecond, plus 1 millisecond min drift
+      # for small TTLs.
+      (val.to_i * CLOCK_DRIFT_FACTOR).to_i + 2
+    end
+
+    #
+    # Checks if the lock has been taken
+    #
+    # @param [Redis] conn a redis connection
+    #
+    # @return [true, false]
+    #
+    def taken?(conn)
+      conn.hexists(key.locked, job_id)
     end
 
-    def current_time
-      seconds, microseconds_with_frac = redis_time
-      Time.at(seconds, microseconds_with_frac)
+    def warn_about_timeout
+      log_warn("Timed out after #{config.timeout}s while waiting for primed token (digest: #{key}, job_id: #{job_id})")
     end
 
-    def redis_time
-      redis(&:time)
+    def lock_info
+      @lock_info ||= dump_json(
+        WORKER => item[CLASS],
+        QUEUE => item[QUEUE],
+        LIMIT => item[LOCK_LIMIT],
+        TIMEOUT => item[LOCK_TIMEOUT],
+        TTL => item[LOCK_TTL],
+        LOCK => config.type,
+        LOCK_ARGS => item[LOCK_ARGS],
+        TIME => now_f,
+      )
     end
   end
-  # rubocop:enable Metrics/ClassLength
 end