sidekiq-unique-jobs 6.0.25 → 7.0.0.beta2

data/lib/sidekiq_unique_jobs/orphans/manager.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Orphans
+    #
+    # Manages the orphan reaper
+    #
+    # @author Mikael Henriksson <mikael@zoolutions.se>
+    #
+    class Manager
+      include SidekiqUniqueJobs::Connection
+      include SidekiqUniqueJobs::Logging
+
+      #
+      # Starts a separate thread that periodically reaps orphans
+      #
+      #
+      # @return [Concurrent::TimerTask] the task that was started
+      #
+      def self.start
+        with_logging_context do
+          logger.info("Starting Reaper")
+          task.add_observer(Observer.new)
+          task.execute
+          task
+        end
+      end
+
+      #
+      # Stops the thread that reaps orphans
+      #
+      #
+      # @return [Boolean]
+      #
+      def self.stop
+        with_logging_context do
+          logger.info("Stopping Reaper")
+          task.shutdown
+        end
+      end
+
+      #
+      # The task that runs the reaper
+      #
+      #
+      # @return [<type>] <description>
+      #
+      def self.task
+        @task ||= Concurrent::TimerTask.new(timer_task_options) do
+          with_logging_context do
+            redis do |conn|
+              Orphans::Reaper.call(conn)
+            end
+          end
+        end
+      end
+
+      #
+      # Arguments passed on to the timer task
+      #
+      #
+      # @return [Hash]
+      #
+      def self.timer_task_options
+        { run_now: true,
+          execution_interval: reaper_interval,
+          timeout_interval: reaper_timeout }
+      end
+
+      #
+      # @see SidekiqUniqueJobs::Config#reaper_interval
+      #
+      def self.reaper_interval
+        SidekiqUniqueJobs.config.reaper_interval
+      end
+
+      #
+      # @see SidekiqUniqueJobs::Config#reaper_timeout
+      #
+      def self.reaper_timeout
+        SidekiqUniqueJobs.config.reaper_timeout
+      end
+
+      #
+      # A context to use for all log entries
+      #
+      #
+      # @return [Hash] when logger responds to `:with_context`
+      # @return [String] when logger does not responds to `:with_context`
+      #
+      def self.logging_context
+        if logger_context_hash?
+          { "uniquejobs" => "reaper" }
+        else
+          "uniquejobs=orphan-reaper"
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/orphans/observer.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Class DeleteOrphans provides deletion of orphaned digests
+  #
+  # @note this is a much slower version of the lua script but does not crash redis
+  #
+  # @author Mikael Henriksson <mikael@zoolutions.se>
+  #
+  module Orphans
+    #
+    # Observes the Orphan::Manager and provides information about each execution
+    #
+    # @author Mikael Henriksson <mikael@zoolutions.se>
+    #
+    class Observer
+      include SidekiqUniqueJobs::Logging
+
+      #
+      # Runs every time the {Manager} executes the TimerTask
+      #   used for logging information about the reaping
+      #
+      # @param [Time] time the time of the execution
+      # @param [Object] result the result of the execution
+      # @param [Exception] ex any error raised from the TimerTask
+      #
+      # @return [<type>] <description>
+      #
+      def update(time, result, ex)
+        if result
+          log_info("(#{time}) Execution successfully returned #{result}")
+        elsif ex.is_a?(Concurrent::TimeoutError)
+          log_warn("(#{time}) Execution timed out")
+        else
+          log_info("(#{time}) Cleanup failed with error #{ex.message}")
+          log_error(ex)
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/orphans/reaper.rb

@@ -0,0 +1,201 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Orphans
+    #
+    # Class DeleteOrphans provides deletion of orphaned digests
+    #
+    # @note this is a much slower version of the lua script but does not crash redis
+    #
+    # @author Mikael Henriksson <mikael@zoolutions.se>
+    #
+    class Reaper
+      include SidekiqUniqueJobs::Connection
+      include SidekiqUniqueJobs::Script::Caller
+      include SidekiqUniqueJobs::Logging
+
+      #
+      # Execute deletion of orphaned digests
+      #
+      # @param [Redis] conn nil a connection to redis
+      #
+      # @return [void]
+      #
+      def self.call(conn = nil)
+        return new(conn).call if conn
+
+        redis { |rcon| new(rcon).call }
+      end
+
+      attr_reader :conn, :digests, :scheduled, :retried
+
+      #
+      # Initialize a new instance of DeleteOrphans
+      #
+      # @param [Redis] conn a connection to redis
+      #
+      def initialize(conn)
+        @conn      = conn
+        @digests   = SidekiqUniqueJobs::Digests.new
+        @scheduled = Redis::SortedSet.new(SCHEDULE)
+        @retried   = Redis::SortedSet.new(RETRY)
+      end
+
+      #
+      # Convenient access to the global configuration
+      #
+      #
+      # @return [SidekiqUniqueJobs::Config]
+      #
+      def config
+        SidekiqUniqueJobs.config
+      end
+
+      #
+      # The reaper that was configured
+      #
+      #
+      # @return [Symbol]
+      #
+      def reaper
+        config.reaper
+      end
+
+      #
+      # The number of locks to reap at a time
+      #
+      #
+      # @return [Integer]
+      #
+      def reaper_count
+        config.reaper_count
+      end
+
+      #
+      # Delete orphaned digests
+      #
+      #
+      # @return [Integer] the number of reaped locks
+      #
+      def call
+        case reaper
+        when :ruby
+          execute_ruby_reaper
+        when :lua
+          execute_lua_reaper
+        else
+          log_fatal(":#{reaper} is invalid for `SidekiqUnqiueJobs.config.reaper`")
+        end
+      end
+
+      #
+      # Executes the ruby reaper
+      #
+      #
+      # @return [Integer] the number of deleted locks
+      #
+      def execute_ruby_reaper
+        BatchDelete.call(orphans, conn)
+      end
+
+      #
+      # Executes the lua reaper
+      #
+      #
+      # @return [Integer] the number of deleted locks
+      #
+      def execute_lua_reaper
+        call_script(
+          :reap_orphans,
+          conn,
+          keys: [SidekiqUniqueJobs::DIGESTS, SidekiqUniqueJobs::SCHEDULE, SidekiqUniqueJobs::RETRY],
+          argv: [reaper_count],
+        )
+      end
+
+      #
+      # Find orphaned digests
+      #
+      #
+      # @return [Array<String>] an array of orphaned digests
+      #
+      def orphans
+        conn.zrevrange(digests.key, 0, -1).each_with_object([]) do |digest, result|
+          next if belongs_to_job?(digest)
+
+          result << digest
+          break if result.size >= reaper_count
+        end
+      end
+
+      #
+      # Checks if the digest has a matching job.
+      #   1. It checks the scheduled set
+      #   2. It checks the retry set
+      #   3. It goes through all queues
+      #
+      #
+      # @param [String] digest the digest to search for
+      #
+      # @return [true] when either of the checks return true
+      # @return [false] when no job was found for this digest
+      #
+      def belongs_to_job?(digest)
+        scheduled?(digest) || retried?(digest) || enqueued?(digest)
+      end
+
+      #
+      # Checks if the digest exists in the Sidekiq::ScheduledSet
+      #
+      # @param [String] digest the current digest
+      #
+      # @return [true] when digest exists in scheduled set
+      #
+      def scheduled?(digest)
+        in_sorted_set?(SCHEDULE, digest)
+      end
+
+      #
+      # Checks if the digest exists in the Sidekiq::RetrySet
+      #
+      # @param [String] digest the current digest
+      #
+      # @return [true] when digest exists in retry set
+      #
+      def retried?(digest)
+        in_sorted_set?(RETRY, digest)
+      end
+
+      #
+      # Checks if the digest exists in a Sidekiq::Queue
+      #
+      # @param [String] digest the current digest
+      #
+      # @return [true] when digest exists in any queue
+      #
+      #
+      def enqueued?(digest)
+        if (result = call_script(:find_digest_in_queues, conn, keys: [digest]))
+          log_debug("#{digest} found in #{result}")
+          true
+        else
+          log_debug("#{digest} NOT found in any queues")
+        end
+      end
+
+      #
+      # Checks a sorted set for the existance of this digest
+      #
+      #
+      # @param [String] key the key for the sorted set
+      # @param [String] digest the digest to scan for
+      #
+      # @return [true] when found
+      # @return [false] when missing
+      #
+      def in_sorted_set?(key, digest)
+        conn.zscan_each(key, match: "*#{digest}*", count: 1).to_a.any?
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/profiler.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Class MethodProfiler provides method level profiling
+  #
+  # @author Mikael Henriksson <mikael@zoolutions.se>
+  #
+  class Profiler
+    def self.patch(klass, methods, name) # rubocop:disable Metrics/MethodLength
+      patches = methods.map do |method_name|
+        <<~RUBY
+          unless defined?(#{method_name}__mp_unpatched)
+            alias_method :#{method_name}__mp_unpatched, :#{method_name}
+            def #{method_name}(*args, &blk)
+              unless prof = Thread.current[:_method_profiler]
+                return #{method_name}__mp_unpatched(*args, &blk)
+              end
+              begin
+                start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+                #{method_name}__mp_unpatched(*args, &blk)
+              ensure
+                data = (prof[:#{name}] ||= {duration: 0.0, calls: 0})
+                data[:duration] += Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
+                data[:calls] += 1
+              end
+            end
+          end
+        RUBY
+      end.join("\n")
+
+      klass.class_eval patches
+    end
+
+    def self.start
+      Thread.current[:_method_profiler] = {
+        __start: Process.clock_gettime(Process::CLOCK_MONOTONIC),
+      }
+    end
+
+    def self.stop
+      finish = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      return unless (data = Thread.current[:_method_profiler])
+
+      Thread.current[:_method_profiler] = nil
+      start = data.delete(:__start)
+      data[:total_duration] = finish - start
+      data
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/redis.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Augments and enhances redis with object oriented methods
+  #
+  # @author Mikael Henriksson <mikael@zoolutions.se>
+  #
+  module Redis
+  end
+end

data/lib/sidekiq_unique_jobs/redis/entity.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Redis
+    #
+    # Class Entity functions as a base class for redis types
+    #
+    # @author Mikael Henriksson <mikael@zoolutions.se>
+    #
+    class Entity
+      # includes "SidekiqUniqueJobs::Logging"
+      # @!parse include SidekiqUniqueJobs::Logging
+      include SidekiqUniqueJobs::Logging
+
+      # includes "SidekiqUniqueJobs::Script::Caller"
+      # @!parse include SidekiqUniqueJobs::Script::Caller
+      include SidekiqUniqueJobs::Script::Caller
+
+      # includes "SidekiqUniqueJobs::JSON"
+      # @!parse include SidekiqUniqueJobs::JSON
+      include SidekiqUniqueJobs::JSON
+
+      # includes "SidekiqUniqueJobs::Timing"
+      # @!parse include SidekiqUniqueJobs::Timing
+      include SidekiqUniqueJobs::Timing
+
+      #
+      # @!attribute [r] key
+      #   @return [String] the redis key for this entity
+      attr_reader :key
+
+      #
+      # Initialize a new Entity
+      #
+      # @param [String] key the redis key for this entity
+      #
+      def initialize(key)
+        @key = key
+      end
+
+      #
+      # Checks if the key for this entity exists in redis
+      #
+      #
+      # @return [true] when exists
+      # @return [false] when not exists
+      #
+      def exist?
+        redis { |conn| conn.exists(key) }
+      end
+
+      #
+      # The number of microseconds until the key expires
+      #
+      #
+      # @return [Integer] expiration in milliseconds
+      #
+      def pttl
+        redis { |conn| conn.pttl(key) }
+      end
+
+      #
+      # The number of seconds until the key expires
+      #
+      #
+      # @return [Integer] expiration in seconds
+      #
+      def ttl
+        redis { |conn| conn.ttl(key) }
+      end
+
+      #
+      # Check if the entity has expiration
+      #
+      #
+      # @return [true] when entity is set to exire
+      # @return [false] when entity isn't expiring
+      #
+      def expires?
+        pttl.positive? || ttl.positive?
+      end
+
+      #
+      # Returns the number of entries in this entity
+      #
+      #
+      # @return [Integer] 0
+      #
+      def count
+        0
+      end
+    end
+  end
+end