sidekiq-unique-jobs 6.0.25 → 7.0.0.beta2

data/lib/sidekiq_unique_jobs/job.rb

@@ -17,9 +17,9 @@ module SidekiqUniqueJobs
     private
 
     def add_timeout_and_expiration(item)
-      calculator                = SidekiqUniqueJobs::Timeout::Calculator.new(item)
-      item[LOCK_TIMEOUT_KEY]    = calculator.lock_timeout
-      item[LOCK_EXPIRATION_KEY] = calculator.lock_expiration
+      calculator = SidekiqUniqueJobs::TimeCalculator.new(item)
+      item[LOCK_TIMEOUT] = calculator.lock_timeout
+      item[LOCK_TTL]     = calculator.lock_ttl
     end
 
     def add_unique_args_and_digest(item)

data/lib/sidekiq_unique_jobs/json.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  # Handles loading and dumping of json
+  #
+  # @author Mikael Henriksson <mikael@zoolutions.se>
+  module JSON
+    module_function
+
+    #
+    # Parses a JSON string into an object
+    #
+    # @param [String] string the object to parse
+    #
+    # @return [Object]
+    #
+    def load_json(string)
+      return unless string && !string.empty?
+
+      ::JSON.parse(string)
+    end
+
+    #
+    # Dumps an object into a JSON string
+    #
+    # @param [Object] object a JSON convertible object
+    #
+    # @return [String] a JSON string
+    #
+    def dump_json(object)
+      ::JSON.generate(object)
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/key.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  # Key class wraps logic dealing with various lock keys
+  #
+  # @author Mikael Henriksson <mikael@zoolutions.se>
+  class Key
+    #
+    # @!attribute [r] digest
+    #   @return [String] the digest key for which keys are created
+    attr_reader :digest
+    #
+    # @!attribute [r] queued
+    #   @return [String] the list key with queued job_id's
+    attr_reader :queued
+    #
+    # @!attribute [r] primed
+    #   @return [String] the list key with primed job_id's
+    attr_reader :primed
+    #
+    # @!attribute [r] locked
+    #   @return [String] the hash key with locked job_id's
+    attr_reader :locked
+    #
+    # @!attribute [r] info
+    #   @return [String] information about the lock
+    attr_reader :info
+    #
+    # @!attribute [r] changelog
+    #   @return [String] the zset with changelog entries
+    attr_reader :changelog
+    #
+    # @!attribute [r] digests
+    #   @return [String] the zset with locked digests
+    attr_reader :digests
+
+    #
+    # Initialize a new Key
+    #
+    # @param [String] digest the digest to use as key
+    #
+    def initialize(digest)
+      @digest    = digest
+      @queued    = suffixed_key("QUEUED")
+      @primed    = suffixed_key("PRIMED")
+      @locked    = suffixed_key("LOCKED")
+      @info      = suffixed_key("INFO")
+      @changelog = CHANGELOGS
+      @digests   = DIGESTS
+    end
+
+    #
+    # Provides the only important information about this keys
+    #
+    #
+    # @return [String]
+    #
+    def to_s
+      digest
+    end
+
+    # @see to_s
+    def inspect
+      digest
+    end
+
+    #
+    # Compares keys by digest
+    #
+    # @param [Key] other the key to compare with
+    #
+    # @return [true, false]
+    #
+    def ==(other)
+      digest == other.digest
+    end
+
+    #
+    # Returns all keys as an ordered array
+    #
+    # @return [Array] an ordered array with all keys
+    #
+    def to_a
+      [digest, queued, primed, locked, info, changelog, digests]
+    end
+
+    private
+
+    def suffixed_key(variable)
+      "#{digest}:#{variable}"
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lock.rb

@@ -0,0 +1,295 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Class Lock provides access to information about a lock
+  #
+  # @author Mikael Henriksson <mikael@zoolutions.se>
+  #
+  class Lock # rubocop:disable Metrics/ClassLength
+    # includes "SidekiqUniqueJobs::Connection"
+    # @!parse include SidekiqUniqueJobs::Connection
+    include SidekiqUniqueJobs::Connection
+
+    # includes "SidekiqUniqueJobs::Timing"
+    # @!parse include SidekiqUniqueJobs::Timing
+    include SidekiqUniqueJobs::Timing
+
+    # includes "SidekiqUniqueJobs::JSON"
+    # @!parse include SidekiqUniqueJobs::JSON
+    include SidekiqUniqueJobs::JSON
+
+    #
+    # @!attribute [r] key
+    #   @return [String] the entity redis key
+    attr_reader :key
+
+    #
+    # Initialize a locked lock
+    #
+    # @param [String] digest a unique digest
+    # @param [String] job_id a sidekiq JID
+    # @param [Hash] lock_info information about the lock
+    #
+    # @return [Lock] a newly lock that has been locked
+    #
+    def self.create(digest, job_id, lock_info = {})
+      lock = new(digest, time: Timing.now_f)
+      lock.lock(job_id, lock_info)
+      lock
+    end
+
+    #
+    # Initialize a new lock
+    #
+    # @param [String, Key] key either a digest or an instance of a {Key}
+    # @param [Timstamp, Float] time nil optional timestamp to initiate this lock with
+    #
+    def initialize(key, time: nil)
+      @key        = get_key(key)
+      @created_at = time.is_a?(Float) ? time : time.to_f
+    end
+
+    #
+    # Locks a job_id
+    #
+    # @note intended only for testing purposez
+    #
+    # @param [String] job_id a sidekiq JID
+    # @param [Hash] lock_info information about the lock
+    #
+    # @return [void]
+    #
+    def lock(job_id, lock_info = {})
+      redis do |conn|
+        conn.multi do
+          conn.set(key.digest, job_id)
+          conn.hset(key.locked, job_id, now_f)
+          info.set(lock_info)
+          conn.zadd(key.digests, now_f, key.digest)
+          conn.zadd(key.changelog, now_f, changelog_json(job_id, "queue.lua", "Queued"))
+          conn.zadd(key.changelog, now_f, changelog_json(job_id, "lock.lua", "Locked"))
+        end
+      end
+    end
+
+    #
+    # Unlock a specific job_id
+    #
+    # @param [String] job_id a sidekiq JID
+    #
+    # @return [true] when job_id was removed
+    # @return [false] when job_id wasn't locked
+    #
+    def unlock(job_id)
+      locked.del(job_id)
+    end
+
+    #
+    # Deletes all the redis keys for this lock
+    #
+    #
+    # @return [Integer] the number of keys deleted in redis
+    #
+    def del
+      redis do |conn|
+        conn.multi do
+          conn.zrem(DIGESTS, key.digest)
+          conn.del(key.digest, key.queued, key.primed, key.locked, key.info)
+        end
+      end
+    end
+
+    #
+    # Returns either the time the lock was initialized with or
+    #   the first changelog entry's timestamp
+    #
+    #
+    # @return [Float] a floaty timestamp represantation
+    #
+    def created_at
+      @created_at ||= changelogs.first&.[]("time")
+    end
+
+    #
+    # Returns all job_id's for this lock
+    #
+    # @note a JID can be present in 3 different places
+    #
+    #
+    # @return [Array<String>] an array with JIDs
+    #
+    def all_jids
+      (queued_jids + primed_jids + locked_jids).uniq
+    end
+
+    #
+    # Returns a collection of locked job_id's
+    #
+    # @param [true, false] with_values false provide the timestamp for the lock
+    #
+    # @return [Hash<String, Float>] when given `with_values: true`
+    # @return [Array<String>] when given `with_values: false`
+    #
+    def locked_jids(with_values: false)
+      locked.entries(with_values: with_values)
+    end
+
+    #
+    # Returns the queued JIDs
+    #
+    #
+    # @return [Array<String>] an array with queued job_ids
+    #
+    def queued_jids
+      queued.entries
+    end
+
+    #
+    # Returns the primed JIDs
+    #
+    #
+    # @return [Array<String>] an array with primed job_ids
+    #
+    def primed_jids
+      primed.entries
+    end
+
+    #
+    # Returns all matching changelog entries for this lock
+    #
+    #
+    # @return [Array<Hash>] an array with changelogs
+    #
+    def changelogs
+      changelog.entries(pattern: "*#{key.digest}*")
+    end
+
+    #
+    # The digest key
+    #
+    # @note Used for exists checks to avoid enqueuing
+    #   the same lock twice
+    #
+    #
+    # @return [Redis::String] a string representation of the key
+    #
+    def digest
+      @digest ||= Redis::String.new(key.digest)
+    end
+
+    #
+    # The queued list
+    #
+    #
+    # @return [Redis::List] for queued JIDs
+    #
+    def queued
+      @queued ||= Redis::List.new(key.queued)
+    end
+
+    #
+    # The primed list
+    #
+    #
+    # @return [Redis::List] for primed JIDs
+    #
+    def primed
+      @primed ||= Redis::List.new(key.primed)
+    end
+
+    #
+    # The locked hash
+    #
+    #
+    # @return [Redis::Hash] for locked JIDs
+    #
+    def locked
+      @locked ||= Redis::Hash.new(key.locked)
+    end
+
+    #
+    # Information about the lock
+    #
+    #
+    # @return [Redis::Hash] with lock information
+    #
+    def info
+      @info ||= LockInfo.new(key.info)
+    end
+
+    #
+    # A sorted set with changelog entries
+    #
+    # @see Changelog for more information
+    #
+    #
+    # @return [Changelog]
+    #
+    def changelog
+      @changelog ||= Changelog.new
+    end
+
+    #
+    # A nicely formatted string with information about this lock
+    #
+    #
+    # @return [String]
+    #
+    def to_s
+      <<~MESSAGE
+        Lock status for #{key}
+
+                  value: #{digest.value}
+                   info: #{info.value}
+            queued_jids: #{queued_jids}
+            primed_jids: #{primed_jids}
+            locked_jids: #{locked_jids}
+             changelogs: #{changelogs}
+      MESSAGE
+    end
+
+    #
+    # @see to_s
+    #
+    def inspect
+      to_s
+    end
+
+    private
+
+    #
+    # Ensure the key is a {Key}
+    #
+    # @param [String, Key] key
+    #
+    # @return [Key]
+    #
+    def get_key(key)
+      if key.is_a?(SidekiqUniqueJobs::Key)
+        key
+      else
+        SidekiqUniqueJobs::Key.new(key)
+      end
+    end
+
+    #
+    # Generate a changelog entry for the given arguments
+    #
+    # @param [String] job_id a sidekiq JID
+    # @param [String] script the name of the script generating this entry
+    # @param [String] message a descriptive message for later review
+    #
+    # @return [String] a JSON string matching the Lua script structure
+    #
+    def changelog_json(job_id, script, message)
+      dump_json(
+        digest: key.digest,
+        job_id: job_id,
+        script: script,
+        message: message,
+        time: now_f,
+      )
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/lock/base_lock.rb

@@ -9,6 +9,17 @@ module SidekiqUniqueJobs
     class BaseLock
       include SidekiqUniqueJobs::Logging
 
+      #
+      # Validates that the sidekiq_options for the worker is valid
+      #
+      # @param [Hash] options the sidekiq_options given to the worker
+      #
+      # @return [void]
+      #
+      def self.validate_options(options = {})
+        Validator.validate(options)
+      end
+
       # @param [Hash] item the Sidekiq job hash
       # @param [Proc] callback the callback to use after unlock
       # @param [Sidekiq::RedisConnection, ConnectionPool] redis_pool the redis connection
@@ -16,21 +27,24 @@ module SidekiqUniqueJobs
         @item       = item
         @callback   = callback
         @redis_pool = redis_pool
+        @attempt    = 0
         add_uniqueness_when_missing # Used to ease testing
       end
 
-      # Handles locking of sidekiq jobs.
-      #   Will call a conflict strategy if lock can't be achieved.
-      # @return [String] the sidekiq job id
-      def lock
-        @attempt = 0
-        return item[JID_KEY] if locked?
-
-        if (token = locksmith.lock(item[LOCK_TIMEOUT_KEY]))
-          token
-        else
-          call_strategy
-        end
+      #
+      # Locks a sidekiq job
+      #
+      # @note Will call a conflict strategy if lock can't be achieved.
+      #
+      # @return [String, nil] the locked jid when properly locked, else nil.
+      #
+      # @yield to the caller when given a block
+      #
+      def lock(&block)
+        # TODO: only use replace strategy when server is executing the lock
+        return call_strategy unless (locked_token = locksmith.lock(&block))
+
+        locked_token
       end
 
       # Execute the job in the Sidekiq server processor
@@ -43,7 +57,7 @@ module SidekiqUniqueJobs
       # @return [String] sidekiq job id when successful
       # @return [false] when unsuccessful
       def unlock
-        locksmith.unlock(item[JID_KEY]) # Only signal to release the lock
+        locksmith.unlock # Only signal to release the lock
       end
 
       # Deletes the job from redis if it is locked.
@@ -61,13 +75,23 @@ module SidekiqUniqueJobs
       # @return [true] when this jid has locked the job
       # @return [false] when this jid has not locked the job
       def locked?
-        locksmith.locked?(item[JID_KEY])
+        locksmith.locked?
+      end
+
+      #
+      # The lock manager/client
+      #
+      # @api private
+      # @return [SidekiqUniqueJobs::Locksmith] the locksmith for this sidekiq job
+      #
+      def locksmith
+        @locksmith ||= SidekiqUniqueJobs::Locksmith.new(item, redis_pool)
       end
 
       private
 
       def add_uniqueness_when_missing
-        return if item.key?(UNIQUE_DIGEST_KEY)
+        return if item.key?(UNIQUE_DIGEST)
 
         # The below should only be done to ease testing
         # in production this will be done by the middleware
@@ -83,53 +107,35 @@ module SidekiqUniqueJobs
         strategy.replace? && attempt < 2
       end
 
-      # The sidekiq job hash
-      # @return [Hash] the Sidekiq job hash
+      # @!attribute [r] item
+      #   @return [Hash<String, Object>] the Sidekiq job hash
       attr_reader :item
-
-      # The sidekiq redis pool
-      # @return [Sidekiq::RedisConnection, ConnectionPool, NilClass] the redis connection
+      # @!attribute [r] redis_pool
+      #   @return [Sidekiq::RedisConnection, ConnectionPool, NilClass] the redis connection
       attr_reader :redis_pool
-
-      # The sidekiq job hash
-      # @return [Proc] the callback to use after unlock
+      # @!attribute [r] callback
+      #   @return [Proc] the block to call after unlock
       attr_reader :callback
-
-      # The current attempt to lock the job
-      # @return [Integer] the numerical value of the attempt
+      # @!attribute [r] attempt
+      #   @return [Integer] the current locking attempt
       attr_reader :attempt
 
-      # The interface to the locking mechanism
-      # @return [SidekiqUniqueJobs::Locksmith]
-      def locksmith
-        @locksmith ||= SidekiqUniqueJobs::Locksmith.new(item, redis_pool)
-      end
-
-      def with_cleanup
-        yield
-      rescue Sidekiq::Shutdown
-        log_info("Sidekiq is shutting down, the job `should` be put back on the queue. Keeping the lock!")
-        raise
-      else
-        unlock_with_callback
-      end
-
       def unlock_with_callback
         return log_warn("might need to be unlocked manually") unless unlock
 
         callback_safely
+        item[JID]
       end
 
       def callback_safely
         callback&.call
-        item[JID_KEY]
       rescue StandardError
         log_warn("unlocked successfully but the #after_unlock callback failed!")
         raise
       end
 
       def strategy
-        @strategy ||= OnConflict.find_strategy(item[ON_CONFLICT_KEY]).new(item)
+        @strategy ||= OnConflict.find_strategy(item[ON_CONFLICT]).new(item, redis_pool)
       end
     end
   end