sidekiq-unique-jobs 6.0.24 → 7.0.4

data/lib/sidekiq_unique_jobs/orphans/manager.rb

@@ -0,0 +1,212 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Orphans
+    #
+    # Manages the orphan reaper
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
+    module Manager
+      module_function
+
+      DRIFT_FACTOR = 0.02
+      REAPERS      = [:ruby, :lua].freeze
+
+      include SidekiqUniqueJobs::Connection
+      include SidekiqUniqueJobs::Logging
+
+      #
+      # Starts a separate thread that periodically reaps orphans
+      #
+      #
+      # @return [SidekiqUniqueJobs::TimerTask] the task that was started
+      #
+      def start(test_task = nil) # rubocop:disable
+        return if disabled?
+        return if registered?
+
+        self.task = test_task || default_task
+
+        with_logging_context do
+          register_reaper_process
+          log_info("Starting Reaper")
+          task.add_observer(Observer.new)
+          task.execute
+          task
+        end
+      end
+
+      #
+      # Stops the thread that reaps orphans
+      #
+      #
+      # @return [Boolean]
+      #
+      def stop
+        return if disabled?
+        return if unregistered?
+
+        with_logging_context do
+          log_info("Stopping Reaper")
+          unregister_reaper_process
+          task.shutdown
+        end
+      end
+
+      #
+      # The task that runs the reaper
+      #
+      #
+      # @return [<type>] <description>
+      #
+      def task
+        @task ||= default_task
+      end
+
+      def default_task
+        SidekiqUniqueJobs::TimerTask.new(timer_task_options) do
+          with_logging_context do
+            redis do |conn|
+              refresh_reaper_mutex
+              Orphans::Reaper.call(conn)
+            end
+          end
+        end
+      end
+
+      def task=(task)
+        @task = task
+      end
+
+      #
+      # Arguments passed on to the timer task
+      #
+      #
+      # @return [Hash]
+      #
+      def timer_task_options
+        { run_now: true,
+          execution_interval: reaper_interval,
+          timeout_interval: reaper_timeout }
+      end
+
+      #
+      # @see SidekiqUniqueJobs::Config#reaper
+      #
+      def reaper
+        SidekiqUniqueJobs.config.reaper
+      end
+
+      #
+      # @see SidekiqUniqueJobs::Config#reaper_interval
+      #
+      def reaper_interval
+        SidekiqUniqueJobs.config.reaper_interval
+      end
+
+      #
+      # @see SidekiqUniqueJobs::Config#reaper_timeout
+      #
+      def reaper_timeout
+        SidekiqUniqueJobs.config.reaper_timeout
+      end
+
+      #
+      # A context to use for all log entries
+      #
+      #
+      # @return [Hash] when logger responds to `:with_context`
+      # @return [String] when logger does not responds to `:with_context`
+      #
+      def logging_context
+        if logger_context_hash?
+          { "uniquejobs" => "reaper" }
+        else
+          "uniquejobs=orphan-reaper"
+        end
+      end
+
+      #
+      # Checks if a reaper is registered
+      #
+      #
+      # @return [true, false]
+      #
+      def registered?
+        redis do |conn|
+          conn.get(UNIQUE_REAPER).to_i + drift_reaper_interval > current_timestamp
+        end
+      end
+
+      #
+      # Checks if that reapers are not registerd
+      #
+      # @see registered?
+      #
+      # @return [true, false]
+      #
+      def unregistered?
+        !registered?
+      end
+
+      #
+      # Checks if reaping is disabled
+      #
+      # @see enabled?
+      #
+      # @return [true, false]
+      #
+      def disabled?
+        !enabled?
+      end
+
+      #
+      # Checks if reaping is enabled
+      #
+      # @return [true, false]
+      #
+      def enabled?
+        REAPERS.include?(reaper)
+      end
+
+      #
+      # Writes a mutex key to redis
+      #
+      #
+      # @return [void]
+      #
+      def register_reaper_process
+        redis { |conn| conn.set(UNIQUE_REAPER, current_timestamp, nx: true, ex: drift_reaper_interval) }
+      end
+
+      #
+      # Updates mutex key
+      #
+      #
+      # @return [void]
+      #
+      def refresh_reaper_mutex
+        redis { |conn| conn.set(UNIQUE_REAPER, current_timestamp, ex: drift_reaper_interval) }
+      end
+
+      #
+      # Removes mutex key from redis
+      #
+      #
+      # @return [void]
+      #
+      def unregister_reaper_process
+        redis { |conn| conn.del(UNIQUE_REAPER) }
+      end
+
+      def drift_reaper_interval
+        reaper_interval + (reaper_interval * DRIFT_FACTOR).to_i
+      end
+
+      def current_timestamp
+        Time.now.to_i
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/orphans/null_reaper.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Orphans
+    #
+    # Class DeleteOrphans provides deletion of orphaned digests
+    #
+    # @note this is a much slower version of the lua script but does not crash redis
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
+    class NullReaper < Reaper
+      #
+      # Delete orphaned digests
+      #
+      #
+      # @return [Integer] the number of reaped locks
+      #
+      def call
+        # NO OP
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/orphans/observer.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Class DeleteOrphans provides deletion of orphaned digests
+  #
+  # @note this is a much slower version of the lua script but does not crash redis
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  module Orphans
+    #
+    # Observes the Orphan::Manager and provides information about each execution
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
+    class Observer
+      include SidekiqUniqueJobs::Logging
+
+      #
+      # Runs every time the {Manager} executes the TimerTask
+      #   used for logging information about the reaping
+      #
+      # @param [Time] time the time of the execution
+      # @param [Object] result the result of the execution
+      # @param [Exception] ex any error raised from the TimerTask
+      #
+      # @return [<type>] <description>
+      #
+      def update(time, result, ex)
+        if result
+          log_info("(#{time}) Execution successfully returned #{result}")
+        elsif ex.is_a?(Concurrent::TimeoutError)
+          log_warn("(#{time}) Execution timed out")
+        else
+          log_info("(#{time}) Cleanup failed with error #{ex.message}")
+          log_error(ex)
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/orphans/reaper.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Orphans
+    #
+    # Class DeleteOrphans provides deletion of orphaned digests
+    #
+    # @note this is a much slower version of the lua script but does not crash redis
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
+    class Reaper
+      include SidekiqUniqueJobs::Connection
+      include SidekiqUniqueJobs::Script::Caller
+      include SidekiqUniqueJobs::Logging
+      include SidekiqUniqueJobs::JSON
+
+      require_relative "lua_reaper"
+      require_relative "ruby_reaper"
+      require_relative "null_reaper"
+
+      #
+      # @return [Hash<Symbol, SidekiqUniqueJobs::Orphans::Reaper] the current implementation of reapers
+      REAPERS = {
+        lua: SidekiqUniqueJobs::Orphans::LuaReaper,
+        ruby: SidekiqUniqueJobs::Orphans::RubyReaper,
+        none: SidekiqUniqueJobs::Orphans::NullReaper,
+        nil => SidekiqUniqueJobs::Orphans::NullReaper,
+        false => SidekiqUniqueJobs::Orphans::NullReaper,
+      }.freeze
+
+      #
+      # Execute deletion of orphaned digests
+      #
+      # @param [Redis] conn nil a connection to redis
+      #
+      # @return [void]
+      #
+      def self.call(conn = nil)
+        return new(conn).call if conn
+
+        redis { |rcon| new(rcon).call }
+      end
+
+      #
+      # @!attribute [r] conn
+      #   @return [Redis] a redis connection
+      attr_reader :conn
+
+      #
+      # Initialize a new instance of DeleteOrphans
+      #
+      # @param [Redis] conn a connection to redis
+      #
+      def initialize(conn)
+        @conn = conn
+      end
+
+      #
+      # Convenient access to the global configuration
+      #
+      #
+      # @return [SidekiqUniqueJobs::Config]
+      #
+      def config
+        SidekiqUniqueJobs.config
+      end
+
+      #
+      # The reaper that was configured
+      #
+      #
+      # @return [Symbol]
+      #
+      def reaper
+        config.reaper
+      end
+
+      #
+      # The configured timeout for the reaper
+      #
+      #
+      # @return [Integer] timeout in seconds
+      #
+      def reaper_timeout
+        config.reaper_timeout
+      end
+
+      #
+      # The number of locks to reap at a time
+      #
+      #
+      # @return [Integer]
+      #
+      def reaper_count
+        config.reaper_count
+      end
+
+      #
+      # Delete orphaned digests
+      #
+      #
+      # @return [Integer] the number of reaped locks
+      #
+      def call
+        if (implementation = REAPERS[reaper])
+          implementation.new(conn).call
+        else
+          log_fatal(":#{reaper} is invalid for `SidekiqUnqiueJobs.config.reaper`")
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb

@@ -0,0 +1,201 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Orphans
+    #
+    # Class DeleteOrphans provides deletion of orphaned digests
+    #
+    # @note this is a much slower version of the lua script but does not crash redis
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
+    class RubyReaper < Reaper
+      #
+      # @!attribute [r] digests
+      #   @return [SidekiqUniqueJobs::Digests] digest collection
+      attr_reader :digests
+      #
+      # @!attribute [r] scheduled
+      #   @return [Redis::SortedSet] the Sidekiq ScheduleSet
+      attr_reader :scheduled
+      #
+      # @!attribute [r] retried
+      #   @return [Redis::SortedSet] the Sidekiq RetrySet
+      attr_reader :retried
+
+      #
+      # Initialize a new instance of DeleteOrphans
+      #
+      # @param [Redis] conn a connection to redis
+      #
+      def initialize(conn)
+        super(conn)
+        @digests   = SidekiqUniqueJobs::Digests.new
+        @scheduled = Redis::SortedSet.new(SCHEDULE)
+        @retried   = Redis::SortedSet.new(RETRY)
+      end
+
+      #
+      # Delete orphaned digests
+      #
+      #
+      # @return [Integer] the number of reaped locks
+      #
+      def call
+        BatchDelete.call(orphans, conn)
+      end
+
+      #
+      # Find orphaned digests
+      #
+      #
+      # @return [Array<String>] an array of orphaned digests
+      #
+      def orphans
+        conn.zrevrange(digests.key, 0, -1).each_with_object([]) do |digest, memo|
+          next if belongs_to_job?(digest)
+
+          memo << digest
+          break if memo.size >= reaper_count
+        end
+      end
+
+      #
+      # Checks if the digest has a matching job.
+      #   1. It checks the scheduled set
+      #   2. It checks the retry set
+      #   3. It goes through all queues
+      #
+      #
+      # @param [String] digest the digest to search for
+      #
+      # @return [true] when either of the checks return true
+      # @return [false] when no job was found for this digest
+      #
+      def belongs_to_job?(digest)
+        scheduled?(digest) || retried?(digest) || enqueued?(digest) || active?(digest)
+      end
+
+      #
+      # Checks if the digest exists in the Sidekiq::ScheduledSet
+      #
+      # @param [String] digest the current digest
+      #
+      # @return [true] when digest exists in scheduled set
+      #
+      def scheduled?(digest)
+        in_sorted_set?(SCHEDULE, digest)
+      end
+
+      #
+      # Checks if the digest exists in the Sidekiq::RetrySet
+      #
+      # @param [String] digest the current digest
+      #
+      # @return [true] when digest exists in retry set
+      #
+      def retried?(digest)
+        in_sorted_set?(RETRY, digest)
+      end
+
+      #
+      # Checks if the digest exists in a Sidekiq::Queue
+      #
+      # @param [String] digest the current digest
+      #
+      # @return [true] when digest exists in any queue
+      #
+      def enqueued?(digest)
+        Sidekiq.redis do |conn|
+          queues(conn) do |queue|
+            entries(conn, queue) do |entry|
+              return true if entry.include?(digest)
+            end
+          end
+
+          false
+        end
+      end
+
+      def active?(digest) # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+        Sidekiq.redis do |conn|
+          procs = conn.sscan_each("processes").to_a
+          return false if procs.empty?
+
+          procs.sort.each do |key|
+            valid, workers = conn.pipelined do
+              conn.exists(key)
+              conn.hgetall("#{key}:workers")
+            end
+
+            next unless valid
+            next unless workers.any?
+
+            workers.each_pair do |_tid, job|
+              next unless (item = safe_load_json(job))
+
+              payload = safe_load_json(item[PAYLOAD])
+
+              return true if payload[LOCK_DIGEST] == digest
+              return true if considered_active?(payload[CREATED_AT])
+            end
+          end
+
+          false
+        end
+      end
+
+      def considered_active?(time_f)
+        (Time.now - reaper_timeout).to_f < time_f
+      end
+
+      #
+      # Loops through all the redis queues and yields them one by one
+      #
+      # @param [Redis] conn the connection to use for fetching queues
+      #
+      # @return [void]
+      #
+      # @yield queues one at a time
+      #
+      def queues(conn, &block)
+        conn.sscan_each("queues", &block)
+      end
+
+      def entries(conn, queue, &block) # rubocop:disable Metrics/MethodLength
+        queue_key    = "queue:#{queue}"
+        initial_size = conn.llen(queue_key)
+        deleted_size = 0
+        page         = 0
+        page_size    = 50
+
+        loop do
+          range_start = page * page_size - deleted_size
+          range_end   = range_start + page_size - 1
+          entries     = conn.lrange(queue_key, range_start, range_end)
+          page       += 1
+
+          break if entries.empty?
+
+          entries.each(&block)
+
+          deleted_size = initial_size - conn.llen(queue_key)
+        end
+      end
+
+      #
+      # Checks a sorted set for the existance of this digest
+      #
+      #
+      # @param [String] key the key for the sorted set
+      # @param [String] digest the digest to scan for
+      #
+      # @return [true] when found
+      # @return [false] when missing
+      #
+      def in_sorted_set?(key, digest)
+        conn.zscan_each(key, match: "*#{digest}*", count: 1).to_a.any?
+      end
+    end
+  end
+end