sidekiq-unique-jobs 6.0.0.rc6 → 6.0.0.rc7

data/lib/sidekiq_unique_jobs/client/middleware.rb

@@ -4,11 +4,20 @@ require 'sidekiq_unique_jobs/server/middleware'
 
 module SidekiqUniqueJobs
   module Client
+    # The unique sidekiq middleware for the client push
+    #
+    # @author Mikael Henriksson <mikael@zoolutions.se>
     class Middleware
       include SidekiqUniqueJobs::Logging
       include OptionsWithFallback
 
-      # :reek:LongParameterList { max_params: 4 }
+      # Calls this client middleware
+      #   Used from Sidekiq.process_single
+      # @param [String] worker_class name of the sidekiq worker class
+      # @param [Hash] item a sidekiq job hash
+      # @param [String] queue name of the queue
+      # @param [Sidekiq::RedisConnection, ConnectionPool] redis_pool the redis connection
+      # @yield when uniqueness is disable or lock successful
       def call(worker_class, item, queue, redis_pool = nil)
         @worker_class = worker_class
         @item         = item
@@ -20,6 +29,8 @@ module SidekiqUniqueJobs
 
       private
 
+      # The sidekiq job hash
+      # @return [Hash] the Sidekiq job hash
       attr_reader :item
 
       def success?

data/lib/sidekiq_unique_jobs/connection.rb

@@ -1,12 +1,16 @@
 # frozen_string_literal: true
 
 module SidekiqUniqueJobs
+  # Shared module for dealing with redis connections
+  #
+  # @author Mikael Henriksson <mikael@zoolutions.se>
   module Connection
     def self.included(base)
       base.send(:extend, self)
     end
 
-    # :reek:UtilityFunction { enabled: false }
+    # Creates a connection to redis
+    # @return [Sidekiq::RedisConnection, ConnectionPool] a connection to redis
     def redis(redis_pool = nil)
       if redis_pool
         redis_pool.with { |conn| yield conn }

data/lib/sidekiq_unique_jobs/constants.rb

@@ -14,6 +14,9 @@ module SidekiqUniqueJobs
   UNIQUE_ARGS_KEY           ||= 'unique_args'
   UNIQUE_DIGEST_KEY         ||= 'unique_digest'
   UNIQUE_KEY                ||= 'unique'
+  UNIQUE_SET                ||= 'unique:keys'
+  LOCK_KEY                  ||= 'lock'
+  ON_CONFLICT_KEY           ||= 'on_conflict'
   UNIQUE_ON_ALL_QUEUES_KEY  ||= 'unique_on_all_queues' # TODO: Remove in v6.1
   UNIQUE_PREFIX_KEY         ||= 'unique_prefix'
 end

data/lib/sidekiq_unique_jobs/digests.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  # Utility module to help manage unique digests in redis.
+  #
+  # @author Mikael Henriksson <mikael@zoolutions.se>
+  module Digests
+    DEFAULT_COUNT = 1_000
+    SCAN_PATTERN  = '*'
+    CHUNK_SIZE    = 100
+
+    include SidekiqUniqueJobs::Logging
+    include SidekiqUniqueJobs::Connection
+    extend self # rubocop:disable Style/ModuleFunction
+
+    # Return unique digests matching pattern
+    #
+    # @param [String] pattern a pattern to match with
+    # @param [Integer] count the maximum number to match
+    # @return [Array<String>] with unique digests
+    def all(pattern: SCAN_PATTERN, count: DEFAULT_COUNT)
+      redis { |conn| conn.sscan_each(UNIQUE_SET, match: pattern, count: count).to_a }
+    end
+
+    # Get a total count of unique digests
+    #
+    # @return [Integer] number of digests
+    def count
+      redis { |conn| conn.scard(UNIQUE_SET) }
+    end
+
+    # Deletes unique digest either by a digest or pattern
+    #
+    # @param [String] digest the full digest to delete
+    # @param [String] pattern a key pattern to match with
+    # @param [Integer] count the maximum number
+    # @raise [ArgumentError] when both pattern and digest are nil
+    # @return [Array<String>] with unique digests
+    def del(digest: nil, pattern: nil, count: DEFAULT_COUNT)
+      return delete_by_pattern(pattern, count: count) if pattern
+      return delete_by_digest(digest) if digest
+
+      raise ArgumentError, 'either digest or pattern need to be provided'
+    end
+
+    private
+
+    # Deletes unique digests by pattern
+    #
+    # @param [String] pattern a key pattern to match with
+    # @param [Integer] count the maximum number
+    # @return [Array<String>] with unique digests
+    def delete_by_pattern(pattern, count: DEFAULT_COUNT)
+      result, elapsed = timed do
+        digests = all(pattern: pattern, count: count)
+        batch_delete(digests)
+        digests.size
+      end
+
+      log_info("#{__method__}(#{pattern}, count: #{count}) completed in #{elapsed}ms")
+
+      result
+    end
+
+    # Get a total count of unique digests
+    #
+    # @param [String] digest a key pattern to match with
+    def delete_by_digest(digest)
+      result, elapsed = timed do
+        Scripts.call(:delete_by_digest, nil, keys: [UNIQUE_SET, digest])
+        count
+      end
+
+      log_info("#{__method__}(#{digest}) completed in #{elapsed}ms")
+
+      result
+    end
+
+    def batch_delete(digests) # rubocop:disable Metrics/MethodLength
+      redis do |conn|
+        digests.each_slice(CHUNK_SIZE) do |chunk|
+          conn.pipelined do
+            chunk.each do |digest|
+              conn.del digest
+              conn.srem(UNIQUE_SET, digest)
+              conn.del("#{digest}:EXISTS")
+              conn.del("#{digest}:GRABBED")
+              conn.del("#{digest}:VERSION")
+              conn.del("#{digest}:AVAILABLE")
+              conn.del("#{digest}:RUN:EXISTS")
+              conn.del("#{digest}:RUN:GRABBED")
+              conn.del("#{digest}:RUN:VERSION")
+              conn.del("#{digest}:RUN:AVAILABLE")
+            end
+          end
+        end
+      end
+    end
+
+    def timed
+      start = current_time
+      result = yield
+      elapsed = (current_time - start).round(2)
+      [result, elapsed]
+    end
+
+    def current_time
+      Time.now
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/exceptions.rb

@@ -1,30 +1,29 @@
 # frozen_string_literal: true
 
 module SidekiqUniqueJobs
-  class LockTimeout < StandardError
-  end
-
-  class RunLockFailed < StandardError
+  # Error raised when a Lua script fails to execute
+  #
+  # @author Mikael Henriksson <mikael@zoolutions.se>
+  class Conflict < StandardError
+    def initialize(item)
+      super("Item with the key: #{item[UNIQUE_DIGEST_KEY]} is already scheduled or processing")
+    end
   end
 
+  # Error raised from {OnConflict::Raise}
+  #
+  # @author Mikael Henriksson <mikael@zoolutions.se>
   class ScriptError < StandardError
+    # @param [Symbol] file_name the name of the lua script
+    # @param [Redis::CommandError] source_exception exception to handle
     def initialize(file_name:, source_exception:)
       super("Problem compiling #{file_name}. Message: #{source_exception.message}")
     end
   end
 
-  class UniqueKeyMissing < ArgumentError
-  end
-
-  class JidMissing < ArgumentError
-  end
-
-  class MaxLockTimeMissing < ArgumentError
-  end
-
-  class UnexpectedValue < StandardError
-  end
-
+  # Error raised from {OptionsWithFallback#lock_class}
+  #
+  # @author Mikael Henriksson <mikael@zoolutions.se>
   class UnknownLock < StandardError
   end
 end

data/lib/sidekiq_unique_jobs/lock/base_lock.rb

@@ -2,43 +2,80 @@
 
 module SidekiqUniqueJobs
   class Lock
+    # Abstract base class for locks
+    #
+    # @abstract
+    # @author Mikael Henriksson <mikael@zoolutions.se>
     class BaseLock
       include SidekiqUniqueJobs::Logging
 
+      # @param [Hash] item the Sidekiq job hash
+      # @param [Proc] callback the callback to use after unlock
+      # @param [Sidekiq::RedisConnection, ConnectionPool] redis_pool the redis connection
       def initialize(item, callback, redis_pool = nil)
         @item       = prepare_item(item)
         @callback   = callback
         @redis_pool = redis_pool
       end
 
+      # Handles locking of sidekiq jobs.
+      #   Will call a conflict strategy if lock can't be achieved.
+      # @return [String] the sidekiq job id
       def lock
-        locksmith.lock(item[LOCK_TIMEOUT_KEY])
+        if (token = locksmith.lock(item[LOCK_TIMEOUT_KEY]))
+          token
+        else
+          strategy.call
+        end
       end
 
+      # Execute the job in the Sidekiq server processor
+      # @raise [NotImplementedError] needs to be implemented in child class
       def execute
         raise NotImplementedError, "##{__method__} needs to be implemented in #{self.class}"
       end
 
+      # Unlocks the job from redis
+      # @return [String] sidekiq job id when successful
+      # @return [false] when unsuccessful
       def unlock
         locksmith.signal(item[JID_KEY]) # Only signal to release the lock
       end
 
+      # Deletes the job from redis if it is locked.
       def delete
         locksmith.delete # Soft delete (don't forcefully remove when expiration is set)
       end
 
+      # Forcefully deletes the job from redis.
+      #   This is good for jobs when a previous lock was not unlocked
       def delete!
         locksmith.delete! # Force delete the lock
       end
 
+      # Checks if the item has achieved a lock
+      # @return [true] when this jid has locked the job
+      # @return [false] when this jid has not locked the job
       def locked?
         locksmith.locked?(item[JID_KEY])
       end
 
       private
 
-      attr_reader :item, :redis_pool, :callback
+      # The sidekiq job hash
+      # @return [Hash] the Sidekiq job hash
+      attr_reader :item
 
+      # The sidekiq redis pool
+      # @return [Sidekiq::RedisConnection, ConnectionPool, NilClass] the redis connection
+      attr_reader :redis_pool
+
+      # The sidekiq job hash
+      # @return [Proc] the callback to use after unlock
+      attr_reader :callback
+
+      # The interface to the locking mechanism
+      # @return [SidekiqUniqueJobs::Locksmith]
       def locksmith
         @locksmith ||= SidekiqUniqueJobs::Locksmith.new(item, redis_pool)
       end
@@ -75,9 +112,13 @@ module SidekiqUniqueJobs
       def callback_safely
         callback&.call
       rescue StandardError
-        log_warn("The lock for #{item[UNIQUE_DIGEST_KEY]} has been released but the #after_unlock callback failed!")
+        log_warn("The unique_key: #{item[UNIQUE_DIGEST_KEY]} has been unlocked but the #after_unlock callback failed!")
         raise
       end
+
+      def strategy
+        @strategy ||= OnConflict.find_strategy(item[ON_CONFLICT_KEY]).new(item)
+      end
     end
   end
 end

data/lib/sidekiq_unique_jobs/lock/until_and_while_executing.rb

@@ -2,14 +2,24 @@
 
 module SidekiqUniqueJobs
   class Lock
+    # Locks jobs while the job is executing in the server process
+    # - Locks on perform_in or perform_async (see {UntilExecuting})
+    # - Unlocks before yielding to the worker's perform method (see {UntilExecuting})
+    # - Locks before yielding to the worker's perform method (see {WhileExecuting})
+    # - Unlocks after yielding to the worker's perform method (see {WhileExecuting})
+    #
+    # See {#lock} for more information about the client.
+    # See {#execute} for more information about the server
+    #
+    # @author Mikael Henriksson <mikael@zoolutions.se>
     class UntilAndWhileExecuting < BaseLock
+      # Executes in the Sidekiq server process
+      # @yield to the worker class perform method
       def execute
         return unless locked?
         unlock
 
-        runtime_lock.execute do
-          yield if block_given?
-        end
+        runtime_lock.execute { yield }
       end
 
       def runtime_lock

data/lib/sidekiq_unique_jobs/lock/until_executed.rb

@@ -2,12 +2,19 @@
 
 module SidekiqUniqueJobs
   class Lock
+    # Locks jobs until the server is done executing the job
+    # - Locks on perform_in or perform_async
+    # - Unlocks after yielding to the worker's perform method
+    #
+    # @author Mikael Henriksson <mikael@zoolutions.se>
     class UntilExecuted < BaseLock
       OK ||= 'OK'
 
+      # Executes in the Sidekiq server process
+      # @yield to the worker class perform method
       def execute
         return unless locked?
-        with_cleanup { yield if block_given? }
+        with_cleanup { yield }
       end
     end
   end

data/lib/sidekiq_unique_jobs/lock/until_executing.rb

@@ -2,10 +2,17 @@
 
 module SidekiqUniqueJobs
   class Lock
+    # Locks jobs until {#execute} starts
+    # - Locks on perform_in or perform_async
+    # - Unlocks after yielding to the worker's perform method
+    #
+    # @author Mikael Henriksson <mikael@zoolutions.se>
     class UntilExecuting < BaseLock
+      # Executes in the Sidekiq server process
+      # @yield to the worker class perform method
       def execute
         unlock_with_callback
-        yield if block_given?
+        yield
       end
     end
   end

data/lib/sidekiq_unique_jobs/lock/until_expired.rb

@@ -2,15 +2,27 @@
 
 module SidekiqUniqueJobs
   class Lock
+    # Locks jobs until the lock has expired
+    # - Locks on perform_in or perform_async
+    # - Unlocks when the expiration is hit
+    #
+    # See {#lock} for more information about the client.
+    # See {#execute} for more information about the server
+    #
+    # @author Mikael Henriksson <mikael@zoolutions.se>
     class UntilExpired < BaseLock
+      # Prevents these locks from being unlocked
+      # @return [true] always returns true
       def unlock
         true
       end
 
+      # Executes in the Sidekiq server process
+      # @yield to the worker class perform method
       def execute
         return unless locked?
-        yield if block_given?
-        # this lock does not handle after_unlock since we don't know when that would
+        yield
+        # this lock does not handle after_unlock since we don't know when that would happen
       end
     end
   end

data/lib/sidekiq_unique_jobs/lock/while_executing.rb

@@ -2,24 +2,38 @@
 
 module SidekiqUniqueJobs
   class Lock
+    # Locks jobs while the job is executing in the server process
+    # - Locks before yielding to the worker's perform method
+    # - Unlocks after yielding to the worker's perform method
+    #
+    # See {#lock} for more information about the client.
+    # See {#execute} for more information about the server
+    #
+    # @author Mikael Henriksson <mikael@zoolutions.se>
     class WhileExecuting < BaseLock
       RUN_SUFFIX ||= ':RUN'
 
+      # @param [Hash] item the Sidekiq job hash
+      # @param [Proc] callback callback to call after unlock
+      # @param [Sidekiq::RedisConnection, ConnectionPool] redis_pool the redis connection
       def initialize(item, callback, redis_pool = nil)
         super(item, callback, redis_pool)
         append_unique_key_suffix
       end
 
-      # Returning true makes sure the client
-      #   can push the job on the queue
+      # Simulate that a client lock was achieved.
+      #   These locks should only ever be created in the server process.
+      # @return [true] always returns true
       def lock
         true
       end
 
-      # Locks the job with the RUN_SUFFIX appended
+      # Executes in the Sidekiq server process.
+      #   These jobs are locked in the server process not from the client
+      # @yield to the worker class perform method
       def execute
-        return unless locksmith.lock(item[LOCK_TIMEOUT_KEY])
-        with_cleanup { yield if block_given? }
+        return strategy.call unless locksmith.lock(item[LOCK_TIMEOUT_KEY])
+        with_cleanup { yield }
       end
 
       private