sidekiq-unique-jobs 4.0.18 → 7.1.28

data/lib/sidekiq_unique_jobs/middleware/client.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Middleware
+    # The unique sidekiq middleware for the client push
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class Client
+      include Sidekiq::ClientMiddleware if defined?(Sidekiq::ClientMiddleware)
+
+      # prepend "SidekiqUniqueJobs::Middleware"
+      # @!parse prepends SidekiqUniqueJobs::Middleware
+      prepend SidekiqUniqueJobs::Middleware
+      # includes "SidekiqUniqueJobs::Reflectable"
+      # @!parse include SidekiqUniqueJobs::Reflectable
+      include SidekiqUniqueJobs::Reflectable
+
+      # Calls this client middleware
+      #   Used from Sidekiq.process_single
+      #
+      # @see SidekiqUniqueJobs::Middleware#call
+      #
+      # @see https://github.com/mperham/sidekiq/wiki/Job-Format
+      # @see https://github.com/mperham/sidekiq/wiki/Middleware
+      #
+      # @yield when uniqueness is disable
+      # @yield when the lock is successful
+      def call(*, &block)
+        lock(&block)
+      end
+
+      private
+
+      def lock
+        lock_instance.lock do
+          reflect(:locked, item)
+          return yield
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/middleware/server.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Middleware
+    # The unique sidekiq middleware for the server processor
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class Server
+      include Sidekiq::ServerMiddleware if defined?(Sidekiq::ServerMiddleware)
+
+      # prepend "SidekiqUniqueJobs::Middleware"
+      # @!parse prepends SidekiqUniqueJobs::Middleware
+      prepend SidekiqUniqueJobs::Middleware
+
+      #
+      #
+      # Runs the server middleware (used from Sidekiq::Processor#process)
+      #
+      # @see SidekiqUniqueJobs::Middleware#call
+      #
+      # @see https://github.com/mperham/sidekiq/wiki/Job-Format
+      # @see https://github.com/mperham/sidekiq/wiki/Middleware
+      #
+      # @yield when uniqueness is disabled
+      # @yield when owning the lock
+      def call(*, &block)
+        lock_instance.execute(&block)
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/middleware.rb

@@ -1,40 +1,45 @@
-require 'sidekiq'
+# frozen_string_literal: true
 
 module SidekiqUniqueJobs
+  #
+  # Provides the sidekiq middleware that makes the gem work
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
   module Middleware
-    def self.extended(base)
-      base.class_eval do
-        configure_middleware
-      end
-    end
+    include SidekiqUniqueJobs::Logging::Middleware
+    include SidekiqUniqueJobs::OptionsWithFallback
+    include SidekiqUniqueJobs::JSON
 
-    def configure_middleware
-      configure_server_middleware
-      configure_client_middleware
-    end
+    # The sidekiq job hash
+    # @return [Hash] the Sidekiq job hash
+    attr_reader :item
 
-    def configure_server_middleware
-      Sidekiq.configure_server do |config|
-        config.client_middleware do |chain|
-          require 'sidekiq_unique_jobs/client/middleware'
-          chain.add SidekiqUniqueJobs::Client::Middleware
-        end
+    #
+    # This method runs before (prepended) the actual middleware implementation.
+    #   This is done to reduce duplication
+    #
+    # @param [Sidekiq::Worker] worker_class
+    # @param [Hash] item a sidekiq job hash
+    # @param [String] queue name of the queue
+    # @param [ConnectionPool] redis_pool only used for compatility reasons
+    #
+    # @return [yield<super>] <description>
+    #
+    # @yieldparam [<type>] if <description>
+    # @yieldreturn [<type>] <describe what yield should return>
+    def call(worker_class, item, queue, redis_pool = nil)
+      @item       = item
+      @queue      = queue
+      @redis_pool = redis_pool
+      self.job_class = item[CLASS]
+      return yield if unique_disabled?
 
-        config.server_middleware do |chain|
-          require 'sidekiq_unique_jobs/server/middleware'
-          chain.add SidekiqUniqueJobs::Server::Middleware
-        end
-      end
-    end
+      SidekiqUniqueJobs::Job.prepare(item) unless item[LOCK_DIGEST]
 
-    def configure_client_middleware
-      Sidekiq.configure_client do |config|
-        config.client_middleware do |chain|
-          require 'sidekiq_unique_jobs/client/middleware'
-          chain.add SidekiqUniqueJobs::Client::Middleware
-        end
+      with_logging_context do
+        super
       end
     end
   end
 end
-SidekiqUniqueJobs.send(:extend, SidekiqUniqueJobs::Middleware)

data/lib/sidekiq_unique_jobs/normalizer.rb

@@ -1,7 +1,17 @@
+# frozen_string_literal: true
+
 module SidekiqUniqueJobs
+  # Normalizes hashes by dumping them to json and loading them from json
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
   module Normalizer
+    extend SidekiqUniqueJobs::JSON
+
+    # Changes hash to a json compatible hash
+    # @param [Hash] args
+    # @return [Hash] a json compatible hash
     def self.jsonify(args)
-      JSON.parse(args.to_json)
+      load_json(dump_json(args))
     end
   end
 end

data/lib/sidekiq_unique_jobs/on_conflict/log.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module OnConflict
+    # Strategy to log information about conflict
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class Log < OnConflict::Strategy
+      include SidekiqUniqueJobs::Logging
+
+      #
+      # Logs an informational message about that the job was not unique
+      #
+      #
+      # @return [void]
+      #
+      def call
+        log_info(<<~MESSAGE.chomp)
+          Skipping job with id (#{item[JID]}) because lock_digest: (#{item[LOCK_DIGEST]}) already exists
+        MESSAGE
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/on_conflict/null_strategy.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module OnConflict
+    # Default conflict strategy class that does nothing
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class NullStrategy < OnConflict::Strategy
+      # Do nothing on conflict
+      # @return [nil]
+      def call
+        # NOOP
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/on_conflict/raise.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module OnConflict
+    # Strategy to raise an error on conflict
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class Raise < OnConflict::Strategy
+      # Raise an error on conflict.
+      #   This will cause Sidekiq to retry the job
+      # @raise [SidekiqUniqueJobs::Conflict]
+      def call
+        raise SidekiqUniqueJobs::Conflict, item
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/on_conflict/reject.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module OnConflict
+    # Strategy to send jobs to dead queue
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class Reject < OnConflict::Strategy
+      include SidekiqUniqueJobs::Timing
+
+      # Send jobs to dead queue
+      def call
+        log_info { "Adding dead #{item[CLASS]} job #{item[JID]}" }
+
+        if deadset_kill?
+          deadset_kill
+        else
+          push_to_deadset
+        end
+      end
+
+      #
+      # Sidekiq version compatibility check
+      # @api private
+      #
+      #
+      # @return [true, false] depending on if Sidekiq::Deadset responds to kill
+      #
+      def deadset_kill?
+        deadset.respond_to?(:kill)
+      end
+
+      #
+      # Use Sidekiqs built in Sidekiq::DeadSet#kill
+      #   to get rid of the job
+      # @api private
+      #
+      #
+      # @return [void]
+      #
+      def deadset_kill
+        if kill_with_options?
+          kill_job_with_options
+        else
+          kill_job_without_options
+        end
+      end
+
+      #
+      # Sidekiq version compatibility check
+      # @api private
+      #
+      #
+      # @return [true] when Sidekiq::Deadset#kill takes more than 1 argument
+      # @return [false] when Sidekiq::Deadset#kill does not take multiple arguments
+      #
+      def kill_with_options?
+        Sidekiq::DeadSet.instance_method(:kill).arity > 1
+      end
+
+      #
+      # Executes the kill instructions without arguments
+      # @api private
+      #
+      # @return [void]
+      #
+      def kill_job_without_options
+        deadset.kill(payload)
+      end
+
+      #
+      # Executes the kill instructions with arguments
+      # @api private
+      #
+      # @return [void]
+      #
+      def kill_job_with_options
+        deadset.kill(payload, notify_failure: false)
+      end
+
+      #
+      # An instance of Sidekiq::Deadset
+      # @api private
+      #
+      # @return [Sidekiq::Deadset]>
+      #
+      def deadset
+        @deadset ||= Sidekiq::DeadSet.new
+      end
+
+      #
+      # Used for compatibility with older Sidekiq versions
+      #
+      #
+      # @return [void]
+      #
+      def push_to_deadset
+        redis do |conn|
+          conn.multi do |pipeline|
+            pipeline.zadd("dead", now_f, payload)
+            pipeline.zremrangebyscore("dead", "-inf", now_f - Sidekiq::DeadSet.timeout)
+            pipeline.zremrangebyrank("dead", 0, -Sidekiq::DeadSet.max_jobs)
+          end
+        end
+      end
+
+      #
+      # The Sidekiq job hash as JSON
+      #
+      #
+      # @return [String] a JSON formatted string
+      #
+      def payload
+        @payload ||= dump_json(item)
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/on_conflict/replace.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module OnConflict
+    # Strategy to replace the job on conflict
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class Replace < OnConflict::Strategy
+      #
+      # @!attribute [r] queue
+      #   @return [String] rthe sidekiq queue this job belongs to
+      attr_reader :queue
+      #
+      # @!attribute [r] lock_digest
+      #   @return [String] the unique digest to use for locking
+      attr_reader :lock_digest
+
+      #
+      # Initialize a new Replace strategy
+      #
+      # @param [Hash] item sidekiq job hash
+      #
+      def initialize(item, redis_pool = nil)
+        super(item, redis_pool)
+        @queue       = item[QUEUE]
+        @lock_digest = item[LOCK_DIGEST]
+      end
+
+      #
+      # Replace the old job in the queue
+      #
+      #
+      # @return [void] <description>
+      #
+      # @yield to retry the lock after deleting the old one
+      #
+      def call(&block)
+        return unless (deleted_job = delete_job_by_digest)
+
+        log_info("Deleted job: #{deleted_job}")
+        if (del_count = delete_lock)
+          log_info("Deleted `#{del_count}` keys for #{lock_digest}")
+        end
+
+        block&.call
+      end
+
+      #
+      # Delete the job from either schedule, retry or the queue
+      #
+      #
+      # @return [String] the deleted job hash
+      # @return [nil] when deleting nothing
+      #
+      def delete_job_by_digest
+        call_script(:delete_job_by_digest,
+                    keys: ["#{QUEUE}:#{queue}", SCHEDULE, RETRY],
+                    argv: [lock_digest])
+      end
+
+      #
+      # Delete the keys belonging to the job
+      #
+      #
+      # @return [Integer] the number of keys deleted
+      #
+      def delete_lock
+        digests.delete_by_digest(lock_digest)
+      end
+
+      #
+      # Access to the {Digests}
+      #
+      #
+      # @return [Digests] and instance with digests
+      #
+      def digests
+        @digests ||= SidekiqUniqueJobs::Digests.new
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module OnConflict
+    # Strategy to reschedule job on conflict
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class Reschedule < OnConflict::Strategy
+      include SidekiqUniqueJobs::SidekiqWorkerMethods
+      include SidekiqUniqueJobs::Logging
+      include SidekiqUniqueJobs::JSON
+      include SidekiqUniqueJobs::Reflectable
+
+      # @param [Hash] item sidekiq job hash
+      def initialize(item, redis_pool = nil)
+        super(item, redis_pool)
+        self.job_class = item[CLASS]
+      end
+
+      # Create a new job from the current one.
+      #   This will mess up sidekiq stats because a new job is created
+      def call
+        if sidekiq_job_class?
+          if job_class.set(queue: item["queue"].to_sym).perform_in(5, *item[ARGS])
+            reflect(:rescheduled, item)
+          else
+            reflect(:reschedule_failed, item)
+          end
+        else
+          reflect(:unknown_sidekiq_worker, item)
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/on_conflict/strategy.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module OnConflict
+    # Abstract conflict strategy class
+    #
+    # @abstract
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class Strategy
+      include SidekiqUniqueJobs::JSON
+      include SidekiqUniqueJobs::Logging
+      include SidekiqUniqueJobs::Script::Caller
+      include SidekiqUniqueJobs::Timing
+
+      # @!attribute [r] item
+      #   @return [Hash] sidekiq job hash
+      attr_reader :item
+      # @!attribute [r] redis_pool
+      #   @return [Sidekiq::RedisConnection, ConnectionPool, NilClass] the redis connection
+      attr_reader :redis_pool
+
+      #
+      # Initialize a new Strategy
+      #
+      # @param [Hash] item sidekiq job hash
+      # @param [ConnectionPool] redis_pool the connection pool instance
+      #
+      def initialize(item, redis_pool = nil)
+        @item       = item
+        @redis_pool = redis_pool
+      end
+
+      # Use strategy on conflict
+      # @raise [NotImplementedError] needs to be implemented in child class
+      def call
+        raise NotImplementedError, "needs to be implemented in child class"
+      end
+
+      #
+      # Check if the strategy is kind of {Replace}
+      #
+      #
+      # @return [true] when the strategy is a {Replace}
+      # @return [false] when the strategy is not a {Replace}
+      #
+      def replace?
+        is_a?(Replace)
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/on_conflict.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require_relative "on_conflict/strategy"
+require_relative "on_conflict/null_strategy"
+require_relative "on_conflict/log"
+require_relative "on_conflict/raise"
+require_relative "on_conflict/reject"
+require_relative "on_conflict/replace"
+require_relative "on_conflict/reschedule"
+
+module SidekiqUniqueJobs
+  #
+  # Provides lock conflict resolutions
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  module OnConflict
+    # A convenience method for using the configured strategies
+    def self.strategies
+      SidekiqUniqueJobs.strategies
+    end
+
+    #
+    # Find a strategy to use for conflicting locks
+    #
+    # @param [Symbol] strategy the key for the strategy
+    #
+    # @return [OnConflict::Strategy] when found
+    # @return [OnConflict::NullStrategy] when no other could be found
+    #
+    def self.find_strategy(strategy)
+      return OnConflict::NullStrategy unless strategy
+
+      strategies.fetch(strategy.to_sym) do
+        SidekiqUniqueJobs.logger.warn(
+          "No matching implementation for strategy: #{strategy}, returning OnConflict::NullStrategy." \
+          " Available strategies are (#{strategies.inspect})",
+        )
+
+        OnConflict::NullStrategy
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/options_with_fallback.rb

@@ -1,55 +1,78 @@
+# frozen_string_literal: true
+
 module SidekiqUniqueJobs
+  # Module containing methods shared between client and server middleware
+  #
+  # Requires the following methods to be defined in the including class
+  #   1. item (required)
+  #   2. options (can be nil)
+  #   3. job_class (required, can be anything)
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
   module OptionsWithFallback
     def self.included(base)
-      base.class_attribute :lock_cache unless base.respond_to?(:lock_cache)
-      base.lock_cache ||= {}
+      base.send(:include, SidekiqUniqueJobs::SidekiqWorkerMethods)
+    end
+
+    # A convenience method for using the configured locks
+    def locks
+      SidekiqUniqueJobs.locks
     end
 
+    # Check if unique has been enabled
+    # @return [true, false] indicate if the gem has been enabled
     def unique_enabled?
-      options[UNIQUE_KEY] || item[UNIQUE_KEY]
+      SidekiqUniqueJobs.enabled? && lock_type
     end
 
+    # Check if unique has been disabled
     def unique_disabled?
       !unique_enabled?
     end
 
-    def log_duplicate_payload?
-      options[LOG_DUPLICATE_KEY] || item[LOG_DUPLICATE_KEY]
-    end
-
-    def lock
-      @lock = lock_class.new(item)
+    #
+    # A new lock for this Sidekiq Job
+    #
+    #
+    # @return [Lock::BaseLock] an instance of a lock implementation
+    #
+    def lock_instance
+      @lock_instance ||= lock_class.new(item, after_unlock_hook, @redis_pool)
     end
 
+    #
+    # Returns the corresponding class for the lock_type
+    #
+    #
+    # @return [Class]
+    #
     def lock_class
-      lock_cache[unique_lock.to_sym] ||= "SidekiqUniqueJobs::Lock::#{unique_lock.to_s.classify}".constantize
-    end
-
-    def unique_lock
-      @unique_lock ||=
-        if options.key?(UNIQUE_KEY) && options[UNIQUE_KEY].to_s == 'true'
-          warn('unique: true is no longer valid. Please set it to the type of lock required like: ' \
-               '`unique: :until_executed`')
-          options[UNIQUE_LOCK_KEY] || SidekiqUniqueJobs.default_lock
-        else
-          lock_type || SidekiqUniqueJobs.default_lock
-        end
+      @lock_class ||= locks.fetch(lock_type.to_sym) do
+        raise UnknownLock, "No implementation for `lock: :#{lock_type}`"
+      end
     end
 
+    #
+    # The type of lock for this worker
+    #
+    #
+    # @return [Symbol]
+    #
     def lock_type
-      lock_type_from(options) || lock_type_from(item)
-    end
-
-    def lock_type_from(hash, key = UNIQUE_KEY)
-      return nil if hash[key].is_a?(TrueClass)
-      hash[key]
+      @lock_type ||= options[LOCK] || item[LOCK]
     end
 
+    #
+    # The default options with any matching keys overridden from worker options
+    #
+    #
+    # @return [Hash<String, Object>]
+    #
     def options
-      @options ||= worker_class.get_sidekiq_options if worker_class.respond_to?(:get_sidekiq_options)
-      @options ||= Sidekiq.default_worker_options
-      @options ||= {}
-      @options &&= @options.stringify_keys
+      @options ||= begin
+        opts = default_job_options.dup
+        opts.merge!(job_options) if sidekiq_job_class?
+        (opts || {}).stringify_keys
+      end
     end
   end
 end