sidekiq-unique-jobs 7.0.13 → 7.1.0

data/lib/sidekiq_unique_jobs/lock/until_executing.rb

@@ -8,10 +8,24 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     class UntilExecuting < BaseLock
+      #
+      # Locks a sidekiq job
+      #
+      # @note Will call a conflict strategy if lock can't be achieved.
+      #
+      # @return [String, nil] the locked jid when properly locked, else nil.
+      #
+      def lock
+        return lock_failed unless (job_id = locksmith.lock)
+        return yield job_id if block_given?
+
+        job_id
+      end
+
       # Executes in the Sidekiq server process
       # @yield to the worker class perform method
       def execute
-        unlock_with_callback
+        callback_safely if locksmith.unlock
         yield
       end
     end

data/lib/sidekiq_unique_jobs/lock/until_expired.rb

@@ -8,6 +8,27 @@ module SidekiqUniqueJobs
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     #
     class UntilExpired < UntilExecuted
+      #
+      # Locks a sidekiq job
+      #
+      # @note Will call a conflict strategy if lock can't be achieved.
+      #
+      # @return [String, nil] the locked jid when properly locked, else nil.
+      #
+      # @yield to the caller when given a block
+      #
+      def lock
+        return lock_failed unless (job_id = locksmith.lock)
+        return yield job_id if block_given?
+
+        job_id
+      end
+
+      # Executes in the Sidekiq server process
+      # @yield to the worker class perform method
+      def execute(&block)
+        locksmith.execute(&block)
+      end
     end
   end
 end

data/lib/sidekiq_unique_jobs/lock/while_executing.rb

@@ -11,7 +11,7 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     class WhileExecuting < BaseLock
-      RUN_SUFFIX = ":RUN"
+      RUN_SUFFIX ||= ":RUN"
 
       include SidekiqUniqueJobs::OptionsWithFallback
       include SidekiqUniqueJobs::Logging::Middleware
@@ -29,7 +29,10 @@ module SidekiqUniqueJobs
       #   These locks should only ever be created in the server process.
       # @return [true] always returns true
       def lock
-        true
+        job_id = item[JID]
+        yield job_id if block_given?
+
+        job_id
       end
 
       # Executes in the Sidekiq server process.
@@ -37,13 +40,13 @@ module SidekiqUniqueJobs
       # @yield to the worker class perform method
       def execute
         with_logging_context do
-          server_strategy&.call unless locksmith.lock do
+          call_strategy(origin: :server) unless locksmith.execute do
             yield
-            callback_safely
+            callback_safely if locksmith.unlock
+          ensure
+            locksmith.unlock
           end
         end
-      ensure
-        locksmith.unlock!
       end
 
       private
@@ -51,7 +54,9 @@ module SidekiqUniqueJobs
       # This is safe as the base_lock always creates a new digest
       #   The append there for needs to be done every time
       def append_unique_key_suffix
-        item[LOCK_DIGEST] = item[LOCK_DIGEST] + RUN_SUFFIX
+        return if (lock_digest = item[LOCK_DIGEST]).end_with?(RUN_SUFFIX)
+
+        item[LOCK_DIGEST] = lock_digest + RUN_SUFFIX
       end
     end
   end

data/lib/sidekiq_unique_jobs/lock_config.rb

@@ -82,7 +82,7 @@ module SidekiqUniqueJobs
     # @return [true,fakse]
     #
     def wait_for_lock?
-      timeout.nil? || timeout.positive?
+      timeout && (timeout.zero? || timeout.positive?)
     end
 
     #

data/lib/sidekiq_unique_jobs/lock_ttl.rb

@@ -71,7 +71,7 @@ module SidekiqUniqueJobs
       ttl ||= item[LOCK_EXPIRATION] # TODO: Deprecate at some point
       ttl ||= worker_options[LOCK_EXPIRATION] # TODO: Deprecate at some point
       ttl ||= SidekiqUniqueJobs.config.lock_ttl
-      ttl && (ttl.to_i + time_until_scheduled)
+      ttl && ttl.to_i + time_until_scheduled
     end
   end
 end

data/lib/sidekiq_unique_jobs/locksmith.rb

@@ -13,6 +13,10 @@ module SidekiqUniqueJobs
     # @!parse include SidekiqUniqueJobs::Logging
     include SidekiqUniqueJobs::Logging
 
+    # includes "SidekiqUniqueJobs::Reflectable"
+    # @!parse include SidekiqUniqueJobs::Reflectable
+    include SidekiqUniqueJobs::Reflectable
+
     # includes "SidekiqUniqueJobs::Timing"
     # @!parse include SidekiqUniqueJobs::Timing
     include SidekiqUniqueJobs::Timing
@@ -77,7 +81,7 @@ module SidekiqUniqueJobs
     # Deletes the lock regardless of if it has a pttl set
     #
     def delete!
-      call_script(:delete, key.to_a, [job_id, config.pttl, config.type, config.limit]).positive?
+      call_script(:delete, key.to_a, [job_id, config.pttl, config.type, config.limit]).to_i.positive?
     end
 
     #
@@ -85,16 +89,23 @@ module SidekiqUniqueJobs
     #
     # @return [String] the Sidekiq job_id that was locked/queued
     #
-    def lock(&block)
+    def lock(wait: nil)
+      method_name = wait ? :primed_async : :primed_sync
       redis(redis_pool) do |conn|
-        return lock_async(conn, &block) if block
-
-        lock_sync(conn) do
+        lock!(conn, method(method_name), wait) do
           return job_id
         end
       end
     end
 
+    def execute(&block)
+      raise SidekiqUniqueJobs::InvalidArgument, "#execute needs a block" unless block
+
+      redis(redis_pool) do |conn|
+        lock!(conn, method(:primed_async), &block)
+      end
+    end
+
     #
     # Removes the lock keys from Redis if locked by the provided jid/token
     #
@@ -114,11 +125,17 @@ module SidekiqUniqueJobs
     # @return [String] Sidekiq job_id (jid) if successful
     #
     def unlock!(conn = nil)
-      call_script(:unlock, key.to_a, argv, conn)
+      call_script(:unlock, key.to_a, argv, conn) do |unlocked_jid|
+        reflect(:debug, :unlocked, item, unlocked_jid) if unlocked_jid == job_id
+
+        unlocked_jid
+      end
     end
 
     # Checks if this instance is considered locked
     #
+    # @param [Sidekiq::RedisConnection, ConnectionPool] conn the redis connection
+    #
     # @return [true, false] true when the :LOCKED hash contains the job_id
     #
     def locked?(conn = nil)
@@ -134,7 +151,7 @@ module SidekiqUniqueJobs
     # @return [String]
     #
     def to_s
-      "Locksmith##{object_id}(digest=#{key} job_id=#{job_id}, locked=#{locked?})"
+      "Locksmith##{object_id}(digest=#{key} job_id=#{job_id} locked=#{locked?})"
     end
 
     #
@@ -159,70 +176,71 @@ module SidekiqUniqueJobs
 
     attr_reader :redis_pool
 
-    def argv
-      [job_id, config.pttl, config.type, config.limit]
-    end
-
-    #
-    # Used for runtime locks that need automatic unlock after yielding
-    #
-    # @param [Redis] conn a redis connection
     #
-    # @return [nil] when lock was not possible
-    # @return [Object] whatever the block returns when lock was acquired
+    # Used to reduce some duplication from the two methods
     #
-    # @yieldparam [String] job_id a Sidekiq JID
+    # @param [Sidekiq::RedisConnection, ConnectionPool] conn the redis connection
+    # @param [Method] primed_method reference to the method to use for getting a primed token
     #
-    def lock_async(conn)
+    # @yieldparam [string] job_id the sidekiq JID
+    # @yieldreturn [void] whatever the calling block returns
+    def lock!(conn, primed_method, wait = nil)
       return yield job_id if locked?(conn)
 
-      enqueue(conn) do
-        primed_async(conn) do
-          locked_token = call_script(:lock, key.to_a, argv, conn)
-          return yield if locked_token == job_id
+      enqueue(conn) do |queued_jid|
+        reflect(:debug, item, queued_jid)
+
+        primed_method.call(conn, wait) do |primed_jid|
+          reflect(:debug, :primed, item, primed_jid)
+
+          locked_jid = call_script(:lock, key.to_a, argv, conn)
+          if locked_jid
+            reflect(:debug, :locked, item, locked_jid)
+            return yield job_id
+          end
         end
       end
-    ensure
-      unlock!(conn)
     end
 
     #
-    # Pops an enqueued token
-    # @note Used for runtime locks to avoid problems with blocking commands
-    #   in current thread
+    # Prepares all the various lock data
     #
     # @param [Redis] conn a redis connection
     #
-    # @return [nil] when lock was not possible
-    # @return [Object] whatever the block returns when lock was acquired
+    # @return [nil] when redis was already prepared for this lock
+    # @return [yield<String>] when successfully enqueued
     #
-    def primed_async(conn)
-      return yield if Concurrent::Promises
-                      .future(conn) { |red_con| pop_queued(red_con) }
-                      .value(add_drift(config.ttl))
+    def enqueue(conn)
+      queued_jid, elapsed = timed do
+        call_script(:queue, key.to_a, argv, conn)
+      end
+
+      return unless queued_jid
+      return unless [job_id, "1"].include?(queued_jid)
 
-      warn_about_timeout
+      validity = config.pttl - elapsed - drift(config.pttl)
+      return unless validity >= 0 || config.pttl.zero?
+
+      write_lock_info(conn)
+      yield job_id
     end
 
     #
-    # Used for non-runtime locks (no block was given)
+    # Pops an enqueued token
+    # @note Used for runtime locks to avoid problems with blocking commands
+    #   in current thread
     #
     # @param [Redis] conn a redis connection
     #
     # @return [nil] when lock was not possible
     # @return [Object] whatever the block returns when lock was acquired
     #
-    # @yieldparam [String] job_id a Sidekiq JID
-    #
-    def lock_sync(conn)
-      return yield if locked?(conn)
+    def primed_async(conn, wait = nil, &block)
+      primed_jid = Concurrent::Promises
+                   .future(conn) { |red_con| pop_queued(red_con, wait) }
+                   .value(add_drift(wait || config.ttl))
 
-      enqueue(conn) do
-        primed_sync(conn) do
-          locked_token = call_script(:lock, key.to_a, argv, conn)
-          return yield locked_token if locked_token
-        end
-      end
+      handle_primed(primed_jid, &block)
     end
 
     #
@@ -234,12 +252,15 @@ module SidekiqUniqueJobs
     # @return [nil] when lock was not possible
     # @return [Object] whatever the block returns when lock was acquired
     #
-    def primed_sync(conn)
-      if (popped_jid = pop_queued(conn))
-        return yield popped_jid
-      end
+    def primed_sync(conn, wait = nil, &block)
+      primed_jid = pop_queued(conn, wait)
+      handle_primed(primed_jid, &block)
+    end
 
-      warn_about_timeout
+    def handle_primed(primed_jid)
+      return yield job_id if [job_id, "1"].include?(primed_jid)
+
+      reflect(:timeout, item) unless config.wait_for_lock?
     end
 
     #
@@ -249,9 +270,9 @@ module SidekiqUniqueJobs
     #
     # @return [String] a previously enqueued token (now taken off the queue)
     #
-    def pop_queued(conn)
-      if config.wait_for_lock?
-        brpoplpush(conn)
+    def pop_queued(conn, wait = nil)
+      if wait || config.wait_for_lock?
+        brpoplpush(conn, wait)
       else
         rpoplpush(conn)
       end
@@ -260,9 +281,10 @@ module SidekiqUniqueJobs
     #
     # @api private
     #
-    def brpoplpush(conn)
+    def brpoplpush(conn, wait = nil)
+      wait ||= config.timeout
       # passing timeout 0 to brpoplpush causes it to block indefinitely
-      conn.brpoplpush(key.queued, key.primed, timeout: config.timeout)
+      conn.brpoplpush(key.queued, key.primed, timeout: wait)
     end
 
     #
@@ -272,27 +294,6 @@ module SidekiqUniqueJobs
       conn.rpoplpush(key.queued, key.primed)
     end
 
-    #
-    # Prepares all the various lock data
-    #
-    # @param [Redis] conn a redis connection
-    #
-    # @return [nil] when redis was already prepared for this lock
-    # @return [yield<String>] when successfully enqueued
-    #
-    def enqueue(conn)
-      queued_token, elapsed = timed do
-        call_script(:queue, key.to_a, argv, conn)
-      end
-
-      validity = config.pttl - elapsed - drift(config.pttl)
-
-      return unless queued_token && (validity >= 0 || config.pttl.zero?)
-
-      write_lock_info(conn)
-      yield queued_token
-    end
-
     #
     # Writes lock information to redis.
     #   The lock information contains information about worker, queue, limit etc.
@@ -335,10 +336,8 @@ module SidekiqUniqueJobs
       conn.hexists(key.locked, job_id)
     end
 
-    def warn_about_timeout
-      return unless config.wait_for_lock?
-
-      log_debug("Timed out after #{config.timeout}s while waiting for primed token (digest: #{key}, job_id: #{job_id})")
+    def argv
+      [job_id, config.pttl, config.type, config.limit]
     end
 
     def lock_info

data/lib/sidekiq_unique_jobs/middleware/client.rb

@@ -6,7 +6,12 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     class Client
+      # prepend "SidekiqUniqueJobs::Middleware"
+      # @!parse prepends SidekiqUniqueJobs::Middleware
       prepend SidekiqUniqueJobs::Middleware
+      # includes "SidekiqUniqueJobs::Reflectable"
+      # @!parse include SidekiqUniqueJobs::Reflectable
+      include SidekiqUniqueJobs::Reflectable
 
       # Calls this client middleware
       #   Used from Sidekiq.process_single
@@ -25,18 +30,11 @@ module SidekiqUniqueJobs
       private
 
       def lock
-        if (_token = lock_instance.lock)
-          yield
-        else
-          warn_about_duplicate
+        lock_instance.lock do |_locked_jid|
+          reflect(:locked, item)
+          return yield
         end
       end
-
-      def warn_about_duplicate
-        return unless log_duplicate?
-
-        log_warn "Already locked with another job_id (#{dump_json(item)})"
-      end
     end
   end
 end

data/lib/sidekiq_unique_jobs/middleware/server.rb

@@ -6,6 +6,8 @@ module SidekiqUniqueJobs
     #
     # @author Mikael Henriksson <mikael@mhenrixon.com>
     class Server
+      # prepend "SidekiqUniqueJobs::Middleware"
+      # @!parse prepends SidekiqUniqueJobs::Middleware
       prepend SidekiqUniqueJobs::Middleware
 
       #

data/lib/sidekiq_unique_jobs/on_conflict/reschedule.rb

@@ -9,6 +9,7 @@ module SidekiqUniqueJobs
       include SidekiqUniqueJobs::SidekiqWorkerMethods
       include SidekiqUniqueJobs::Logging
       include SidekiqUniqueJobs::JSON
+      include SidekiqUniqueJobs::Reflectable
 
       # @param [Hash] item sidekiq job hash
       def initialize(item, redis_pool = nil)
@@ -20,10 +21,13 @@ module SidekiqUniqueJobs
       #   This will mess up sidekiq stats because a new job is created
       def call
         if sidekiq_worker_class?
-          log_info("Rescheduling #{item[LOCK_DIGEST]}")
-          worker_class.perform_in(5, *item[ARGS])
+          if worker_class.perform_in(5, *item[ARGS])
+            reflect(:rescheduled, item)
+          else
+            reflect(:reschedule_failed, item)
+          end
         else
-          log_warn("Skip rescheduling of #{item[LOCK_DIGEST]} because #{worker_class} is not a Sidekiq::Worker")
+          reflect(:unknown_sidekiq_worker, item)
         end
       end
     end

data/lib/sidekiq_unique_jobs/options_with_fallback.rb

@@ -29,15 +29,6 @@ module SidekiqUniqueJobs
       !unique_enabled?
     end
 
-    # Should duplicate payloads be logged?
-    #
-    #
-    # @return [true, false, nil]
-    #
-    def log_duplicate?
-      options[LOG_DUPLICATE] || item[LOG_DUPLICATE]
-    end
-
     #
     # A new lock for this Sidekiq Job
     #
@@ -55,8 +46,10 @@ module SidekiqUniqueJobs
     # @return [Class]
     #
     def lock_class
-      @lock_class ||= locks.fetch(lock_type.to_sym) do
-        raise UnknownLock, "No implementation for `lock: :#{lock_type}`"
+      @lock_class ||= begin
+        locks.fetch(lock_type.to_sym) do
+          raise UnknownLock, "No implementation for `lock: :#{lock_type}`"
+        end
       end
     end
 

data/lib/sidekiq_unique_jobs/orphans/manager.rb

@@ -31,6 +31,7 @@ module SidekiqUniqueJobs
         with_logging_context do
           register_reaper_process
           log_info("Starting Reaper")
+
           task.add_observer(Observer.new)
           task.execute
           task

data/lib/sidekiq_unique_jobs/orphans/reaper_resurrector.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Orphans
+    # Restarts orphan manager if it is considered dead
+    module ReaperResurrector
+      module_function
+
+      include SidekiqUniqueJobs::Connection
+      include SidekiqUniqueJobs::Logging
+
+      DRIFT_FACTOR = 0.1
+      REAPERS = [:ruby, :lua].freeze
+
+      #
+      # Starts reaper resurrector that watches orphans reaper
+      #
+      # @return [SidekiqUniqueJobs::TimerTask] the task that was started
+      #
+      def start
+        return if resurrector_disabled?
+        return if reaper_disabled?
+
+        with_logging_context do
+          run_task
+        end
+      end
+
+      #
+      # Runs reaper resurrector task
+      #
+      # @return [SidekiqUniqueJobs::TimerTask]
+      def run_task
+        log_info("Starting Reaper Resurrector")
+        task.execute
+        task
+      end
+
+      #
+      # The task that runs the resurrector
+      #
+      # @return [SidekiqUniqueJobs::TimerTask]
+      def task
+        SidekiqUniqueJobs::TimerTask.new(timer_task_options) do
+          with_logging_context do
+            restart_if_dead
+          end
+        end
+      end
+
+      #
+      # Starts new instance of orphan reaper if reaper is considered dead (reaper mutex has not been refreshed lately)
+      #
+      def restart_if_dead
+        return if reaper_registered?
+
+        log_info("Reaper is considered dead. Starting new reaper instance")
+        orphans_manager.start
+      end
+
+      #
+      # Returns orphan manager
+      #
+      # @return [SidekiqUniqueJobs::Orphans::Manager]
+      def orphans_manager
+        SidekiqUniqueJobs::Orphans::Manager
+      end
+
+      #
+      # Checks if resurrector is disabled
+      #
+      # @see resurrector_enabled?
+      #
+      # @return [true, false]
+      def resurrector_disabled?
+        !resurrector_enabled?
+      end
+
+      #
+      # Checks if resurrector is enabled
+      #
+      # @return [true, false]
+      def resurrector_enabled?
+        SidekiqUniqueJobs.config.reaper_resurrector_enabled
+      end
+
+      #
+      # Checks if reaping is disabled
+      #
+      # @see reaper_enabled?
+      #
+      # @return [true, false]
+      #
+      def reaper_disabled?
+        !reaper_enabled?
+      end
+
+      #
+      # Checks if reaping is enabled
+      #
+      # @return [true, false]
+      #
+      def reaper_enabled?
+        REAPERS.include?(reaper)
+      end
+
+      #
+      # Checks if reaper is registered
+      #
+      # @return [true, false]
+      def reaper_registered?
+        redis do |conn|
+          conn.get(UNIQUE_REAPER).to_i + drift_reaper_interval > current_timestamp
+        end
+      end
+
+      #
+      # @see SidekiqUniqueJobs::Config#reaper
+      #
+      def reaper
+        SidekiqUniqueJobs.config.reaper
+      end
+
+      #
+      # Arguments passed on to the timer task
+      #
+      #
+      # @return [Hash]
+      #
+      def timer_task_options
+        { run_now: false,
+          execution_interval: reaper_resurrector_interval }
+      end
+
+      #
+      # A context to use for all log entries
+      #
+      #
+      # @return [Hash] when logger responds to `:with_context`
+      # @return [String] when logger does not responds to `:with_context`
+      #
+      def logging_context
+        if logger_context_hash?
+          { "uniquejobs" => "reaper-resurrector" }
+        else
+          "uniquejobs=reaper-resurrector"
+        end
+      end
+
+      #
+      # @see SidekiqUniqueJobs::Config#reaper_resurrector_interval
+      #
+      def reaper_resurrector_interval
+        SidekiqUniqueJobs.config.reaper_resurrector_interval
+      end
+
+      def reaper_interval
+        SidekiqUniqueJobs.config.reaper_interval
+      end
+
+      def drift_reaper_interval
+        reaper_interval + (reaper_interval * DRIFT_FACTOR).to_i
+      end
+
+      def current_timestamp
+        Time.now.to_i
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb

@@ -182,7 +182,7 @@ module SidekiqUniqueJobs
         page_size    = 50
 
         loop do
-          range_start = (page * page_size) - deleted_size
+          range_start = page * page_size - deleted_size
           range_end   = range_start + page_size - 1
           entries     = conn.lrange(queue_key, range_start, range_end)
           page       += 1

data/lib/sidekiq_unique_jobs/redis/sorted_set.rb

@@ -26,7 +26,7 @@ module SidekiqUniqueJobs
       #
       # Adds a value to the sorted set
       #
-      # @param [Array<Float, String>, String] the values to add
+      # @param [Array<Float, String>, String] values the values to add
       #
       # @return [Boolean, Integer] <description>
       #