sidekiq-unique-jobs 3.0.11 → 8.0.10

data/lib/sidekiq_unique_jobs/middleware/client.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Middleware
+    # The unique sidekiq middleware for the client push
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class Client
+      include Sidekiq::ClientMiddleware
+
+      # prepend "SidekiqUniqueJobs::Middleware"
+      # @!parse prepends SidekiqUniqueJobs::Middleware
+      prepend SidekiqUniqueJobs::Middleware
+      # includes "SidekiqUniqueJobs::Reflectable"
+      # @!parse include SidekiqUniqueJobs::Reflectable
+      include SidekiqUniqueJobs::Reflectable
+
+      # Calls this client middleware
+      #   Used from Sidekiq.process_single
+      #
+      # @see SidekiqUniqueJobs::Middleware#call
+      #
+      # @see https://github.com/mperham/sidekiq/wiki/Job-Format
+      # @see https://github.com/mperham/sidekiq/wiki/Middleware
+      #
+      # @yield when uniqueness is disable
+      # @yield when the lock is successful
+      def call(*, &block)
+        lock(&block)
+      end
+
+      private
+
+      def lock
+        lock_instance.lock do
+          reflect(:locked, item)
+          return yield
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/middleware/server.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Middleware
+    # The unique sidekiq middleware for the server processor
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class Server
+      include Sidekiq::ServerMiddleware
+
+      # prepend "SidekiqUniqueJobs::Middleware"
+      # @!parse prepends SidekiqUniqueJobs::Middleware
+      prepend SidekiqUniqueJobs::Middleware
+
+      #
+      #
+      # Runs the server middleware (used from Sidekiq::Processor#process)
+      #
+      # @see SidekiqUniqueJobs::Middleware#call
+      #
+      # @see https://github.com/mperham/sidekiq/wiki/Job-Format
+      # @see https://github.com/mperham/sidekiq/wiki/Middleware
+      #
+      # @yield when uniqueness is disabled
+      # @yield when owning the lock
+      def call(*, &block)
+        lock_instance.execute(&block)
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/middleware.rb

@@ -1,19 +1,45 @@
-require 'sidekiq'
+# frozen_string_literal: true
 
-Sidekiq.configure_server do |config|
-  config.server_middleware do |chain|
-    require 'sidekiq_unique_jobs/middleware/server/unique_jobs'
-    chain.add SidekiqUniqueJobs::Middleware::Server::UniqueJobs
-  end
-  config.client_middleware do |chain|
-    require 'sidekiq_unique_jobs/middleware/client/unique_jobs'
-    chain.add SidekiqUniqueJobs::Middleware::Client::UniqueJobs
-  end
-end
+module SidekiqUniqueJobs
+  #
+  # Provides the sidekiq middleware that makes the gem work
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  module Middleware
+    include SidekiqUniqueJobs::Logging::Middleware
+    include SidekiqUniqueJobs::OptionsWithFallback
+    include SidekiqUniqueJobs::JSON
+
+    # The sidekiq job hash
+    # @return [Hash] the Sidekiq job hash
+    attr_reader :item
+
+    #
+    # This method runs before (prepended) the actual middleware implementation.
+    #   This is done to reduce duplication
+    #
+    # @param [Sidekiq::Job] worker_class
+    # @param [Hash] item a sidekiq job hash
+    # @param [String] queue name of the queue
+    # @param [ConnectionPool] redis_pool only used for compatility reasons
+    #
+    # @return [yield<super>] call the rest of the middleware stack
+    #
+    # @yieldparam [void] if uniquejobs is disable
+    # @yieldreturn [void] delegate back to other sidekiq middleware
+    def call(worker_class, item, queue, redis_pool = nil)
+      @item       = item
+      @queue      = queue
+      @redis_pool = redis_pool
+      self.job_class = worker_class
+      return yield if unique_disabled?
+
+      SidekiqUniqueJobs::Job.prepare(item)
 
-Sidekiq.configure_client do |config|
-  config.client_middleware do |chain|
-    require 'sidekiq_unique_jobs/middleware/client/unique_jobs'
-    chain.add SidekiqUniqueJobs::Middleware::Client::UniqueJobs
+      with_logging_context do
+        super
+      end
+    end
   end
 end

data/lib/sidekiq_unique_jobs/normalizer.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  # Normalizes hashes by dumping them to json and loading them from json
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  module Normalizer
+    extend SidekiqUniqueJobs::JSON
+
+    # Changes hash to a json compatible hash
+    # @param [Hash] args
+    # @return [Hash] a json compatible hash
+    def self.jsonify(args)
+      load_json(dump_json(args))
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/on_conflict/log.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module OnConflict
+    # Strategy to log information about conflict
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class Log < OnConflict::Strategy
+      include SidekiqUniqueJobs::Logging
+
+      #
+      # Logs an informational message about that the job was not unique
+      #
+      #
+      # @return [void]
+      #
+      def call
+        log_info(<<~MESSAGE.chomp)
+          Skipping job with id (#{item[JID]}) because lock_digest: (#{item[LOCK_DIGEST]}) already exists
+        MESSAGE
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/on_conflict/null_strategy.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module OnConflict
+    # Default conflict strategy class that does nothing
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class NullStrategy < OnConflict::Strategy
+      # Do nothing on conflict
+      # @return [nil]
+      def call
+        # NOOP
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/on_conflict/raise.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module OnConflict
+    # Strategy to raise an error on conflict
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class Raise < OnConflict::Strategy
+      # Raise an error on conflict.
+      #   This will cause Sidekiq to retry the job
+      # @raise [SidekiqUniqueJobs::Conflict]
+      def call
+        raise SidekiqUniqueJobs::Conflict, item
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/on_conflict/reject.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module OnConflict
+    # Strategy to send jobs to dead queue
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class Reject < OnConflict::Strategy
+      include SidekiqUniqueJobs::Timing
+
+      # Send jobs to dead queue
+      def call
+        log_info { "Adding dead #{item[CLASS]} job #{item[JID]}" }
+
+        if kill_with_options?
+          kill_job_with_options
+        else
+          kill_job_without_options
+        end
+      end
+
+      #
+      # Sidekiq version compatibility check
+      # @api private
+      #
+      #
+      # @return [true] when Sidekiq::Deadset#kill takes more than 1 argument
+      # @return [false] when Sidekiq::Deadset#kill does not take multiple arguments
+      #
+      def kill_with_options?
+        Sidekiq::DeadSet.instance_method(:kill).arity > 1
+      end
+
+      #
+      # Executes the kill instructions without arguments
+      # @api private
+      #
+      # @return [void]
+      #
+      def kill_job_without_options
+        deadset.kill(payload)
+      end
+
+      #
+      # Executes the kill instructions with arguments
+      # @api private
+      #
+      # @return [void]
+      #
+      def kill_job_with_options
+        deadset.kill(payload, notify_failure: false)
+      end
+
+      #
+      # An instance of Sidekiq::Deadset
+      # @api private
+      #
+      # @return [Sidekiq::Deadset]>
+      #
+      def deadset
+        @deadset ||= Sidekiq::DeadSet.new
+      end
+
+      #
+      # The Sidekiq job hash as JSON
+      #
+      #
+      # @return [String] a JSON formatted string
+      #
+      def payload
+        @payload ||= dump_json(item)
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/on_conflict/replace.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module OnConflict
+    # Strategy to replace the job on conflict
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class Replace < OnConflict::Strategy
+      #
+      # @!attribute [r] queue
+      #   @return [String] rthe sidekiq queue this job belongs to
+      attr_reader :queue
+      #
+      # @!attribute [r] lock_digest
+      #   @return [String] the unique digest to use for locking
+      attr_reader :lock_digest
+
+      #
+      # Initialize a new Replace strategy
+      #
+      # @param [Hash] item sidekiq job hash
+      #
+      def initialize(item, redis_pool = nil)
+        super(item, redis_pool)
+        @queue       = item[QUEUE]
+        @lock_digest = item[LOCK_DIGEST]
+      end
+
+      #
+      # Replace the old job in the queue
+      #
+      #
+      # @return [void] <description>
+      #
+      # @yield to retry the lock after deleting the old one
+      #
+      def call(&block)
+        return unless (deleted_job = delete_job_by_digest)
+
+        log_info("Deleted job: #{deleted_job}")
+        if (del_count = delete_lock)
+          log_info("Deleted `#{del_count}` keys for #{lock_digest}")
+        end
+
+        block&.call
+      end
+
+      #
+      # Delete the job from either schedule, retry or the queue
+      #
+      #
+      # @return [String] the deleted job hash
+      # @return [nil] when deleting nothing
+      #
+      def delete_job_by_digest
+        call_script(:delete_job_by_digest,
+                    keys: ["#{QUEUE}:#{queue}", SCHEDULE, RETRY],
+                    argv: [lock_digest])
+      end
+
+      #
+      # Delete the keys belonging to the job
+      #
+      #
+      # @return [Integer] the number of keys deleted
+      #
+      def delete_lock
+        digests.delete_by_digest(lock_digest, runtime: false)
+      end
+
+      #
+      # Access to the {Digests}
+      #
+      #
+      # @return [Digests] and instance with digests
+      #
+      def digests
+        @digests ||= SidekiqUniqueJobs::Digests.new
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module OnConflict
+    # Strategy to reschedule job on conflict
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class Reschedule < OnConflict::Strategy
+      include SidekiqUniqueJobs::SidekiqWorkerMethods
+      include SidekiqUniqueJobs::Logging
+      include SidekiqUniqueJobs::JSON
+      include SidekiqUniqueJobs::Reflectable
+
+      # @param [Hash] item sidekiq job hash
+      def initialize(item, redis_pool = nil)
+        super(item, redis_pool)
+        self.job_class = item[CLASS]
+      end
+
+      # Create a new job from the current one.
+      #   This will mess up sidekiq stats because a new job is created
+      def call
+        if sidekiq_job_class?
+          if job_class.set(queue: item["queue"].to_sym).perform_in(schedule_in, *item[ARGS])
+            reflect(:rescheduled, item)
+          else
+            reflect(:reschedule_failed, item)
+          end
+        else
+          reflect(:unknown_sidekiq_worker, item)
+        end
+      end
+
+      def schedule_in
+        job_class.get_sidekiq_options["schedule_in"] || 5
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/on_conflict/strategy.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module OnConflict
+    # Abstract conflict strategy class
+    #
+    # @abstract
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    class Strategy
+      include SidekiqUniqueJobs::JSON
+      include SidekiqUniqueJobs::Logging
+      include SidekiqUniqueJobs::Script::Caller
+      include SidekiqUniqueJobs::Timing
+
+      # @!attribute [r] item
+      #   @return [Hash] sidekiq job hash
+      attr_reader :item
+      # @!attribute [r] redis_pool
+      #   @return [Sidekiq::RedisConnection, ConnectionPool, NilClass] the redis connection
+      attr_reader :redis_pool
+
+      #
+      # Initialize a new Strategy
+      #
+      # @param [Hash] item sidekiq job hash
+      # @param [ConnectionPool] redis_pool the connection pool instance
+      #
+      def initialize(item, redis_pool = nil)
+        @item       = item
+        @redis_pool = redis_pool
+      end
+
+      # Use strategy on conflict
+      # @raise [NotImplementedError] needs to be implemented in child class
+      def call
+        raise NotImplementedError, "needs to be implemented in child class"
+      end
+
+      #
+      # Check if the strategy is kind of {Replace}
+      #
+      #
+      # @return [true] when the strategy is a {Replace}
+      # @return [false] when the strategy is not a {Replace}
+      #
+      def replace?
+        is_a?(Replace)
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/on_conflict.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require_relative "on_conflict/strategy"
+require_relative "on_conflict/null_strategy"
+require_relative "on_conflict/log"
+require_relative "on_conflict/raise"
+require_relative "on_conflict/reject"
+require_relative "on_conflict/replace"
+require_relative "on_conflict/reschedule"
+
+module SidekiqUniqueJobs
+  #
+  # Provides lock conflict resolutions
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  module OnConflict
+    # A convenience method for using the configured strategies
+    def self.strategies
+      SidekiqUniqueJobs.strategies
+    end
+
+    #
+    # Find a strategy to use for conflicting locks
+    #
+    # @param [Symbol] strategy the key for the strategy
+    #
+    # @return [OnConflict::Strategy] when found
+    # @return [OnConflict::NullStrategy] when no other could be found
+    #
+    def self.find_strategy(strategy)
+      return OnConflict::NullStrategy unless strategy
+
+      strategies.fetch(strategy.to_sym) do
+        SidekiqUniqueJobs.logger.warn(
+          "No matching implementation for strategy: #{strategy}, returning OnConflict::NullStrategy." \
+          " Available strategies are (#{strategies.inspect})",
+        )
+
+        OnConflict::NullStrategy
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/options_with_fallback.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  # Module containing methods shared between client and server middleware
+  #
+  # Requires the following methods to be defined in the including class
+  #   1. item (required)
+  #   2. options (can be nil)
+  #   3. job_class (required, can be anything)
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  module OptionsWithFallback
+    def self.included(base)
+      base.send(:include, SidekiqUniqueJobs::SidekiqWorkerMethods)
+    end
+
+    # A convenience method for using the configured locks
+    def locks
+      SidekiqUniqueJobs.locks
+    end
+
+    # Check if unique has been enabled
+    # @return [true, false] indicate if the gem has been enabled
+    def unique_enabled?
+      SidekiqUniqueJobs.enabled? && lock_type
+    end
+
+    # Check if unique has been disabled
+    def unique_disabled?
+      !unique_enabled?
+    end
+
+    #
+    # A new lock for this Sidekiq Job
+    #
+    #
+    # @return [Lock::BaseLock] an instance of a lock implementation
+    #
+    def lock_instance
+      @lock_instance ||= lock_class.new(item, after_unlock_hook, @redis_pool)
+    end
+
+    #
+    # Returns the corresponding class for the lock_type
+    #
+    #
+    # @return [Class]
+    #
+    def lock_class
+      @lock_class ||= locks.fetch(lock_type.to_sym) do
+        raise UnknownLock, "No implementation for `lock: :#{lock_type}`"
+      end
+    end
+
+    #
+    # The type of lock for this worker
+    #
+    #
+    # @return [Symbol, NilClass]
+    #
+    def lock_type
+      @lock_type ||= options[LOCK] || item[LOCK]
+    end
+
+    #
+    # The default options with any matching keys overridden from worker options
+    #
+    #
+    # @return [Hash<String, Object>]
+    #
+    def options
+      @options ||= begin
+        opts = default_job_options.dup
+        opts.merge!(job_options) if sidekiq_job_class?
+        (opts || {}).stringify_keys
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/orphans/lua_reaper.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Orphans
+    #
+    # Class DeleteOrphans provides deletion of orphaned digests
+    #
+    # @note this is a much slower version of the lua script but does not crash redis
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
+    class LuaReaper < Reaper
+      #
+      # Delete orphaned digests
+      #
+      #
+      # @return [Integer] the number of reaped locks
+      #
+      def call
+        call_script(
+          :reap_orphans,
+          conn,
+          keys: [DIGESTS, EXPIRING_DIGESTS, SCHEDULE, RETRY, PROCESSES],
+          argv: [reaper_count, (Time.now - reaper_timeout).to_f],
+        )
+      end
+    end
+  end
+end