sidekiq-unique-jobs 3.0.11 → 8.0.10

data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb

@@ -0,0 +1,313 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Orphans
+    #
+    # Class DeleteOrphans provides deletion of orphaned digests
+    #
+    # @note this is a much slower version of the lua script but does not crash redis
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
+    # rubocop:disable Metrics/ClassLength
+    class RubyReaper < Reaper
+      include SidekiqUniqueJobs::Timing
+
+      #
+      # @return [Integer] a best guess of Sidekiq::Launcher::BEAT_PAUSE
+      SIDEKIQ_BEAT_PAUSE = 10
+      #
+      # @return [String] the suffix for :RUN locks
+      RUN_SUFFIX = ":RUN"
+      #
+      # @return [Integer] the maximum combined length of sidekiq queues for running the reaper
+      MAX_QUEUE_LENGTH = 1000
+      #
+      # @!attribute [r] digests
+      #   @return [SidekiqUniqueJobs::Digests] digest collection
+      attr_reader :digests
+      #
+      # @!attribute [r] scheduled
+      #   @return [Redis::SortedSet] the Sidekiq ScheduleSet
+      attr_reader :scheduled
+      #
+      # @!attribute [r] retried
+      #   @return [Redis::SortedSet] the Sidekiq RetrySet
+      attr_reader :retried
+
+      #
+      # @!attribute [r] start_time
+      #   @return [Integer] The timestamp this execution started represented as Time (used for locks)
+      attr_reader :start_time
+
+      #
+      # @!attribute [r] start_time
+      #   @return [Integer] The clock stamp this execution started represented as integer
+      #      (used for redis compatibility as it is more accurate than time)
+      attr_reader :start_source
+
+      #
+      # @!attribute [r] timeout_ms
+      #   @return [Integer] The allowed ms before timeout
+      attr_reader :timeout_ms
+
+      #
+      # Initialize a new instance of DeleteOrphans
+      #
+      # @param [Redis] conn a connection to redis
+      #
+      def initialize(conn)
+        super(conn)
+        @digests      = SidekiqUniqueJobs::Digests.new
+        @scheduled    = Redis::SortedSet.new(SCHEDULE)
+        @retried      = Redis::SortedSet.new(RETRY)
+        @start_time   = Time.now
+        @start_source = time_source.call
+        @timeout_ms   = SidekiqUniqueJobs.config.reaper_timeout * 1000
+      end
+
+      #
+      # Delete orphaned digests
+      #
+      #
+      # @return [Integer] the number of reaped locks
+      #
+      def call
+        return if queues_very_full?
+
+        BatchDelete.call(expired_digests, conn)
+        BatchDelete.call(orphans, conn)
+
+        # orphans.each_slice(500) do |chunk|
+        #   conn.pipelined do |pipeline|
+        #     chunk.each do |digest|
+        #       next if belongs_to_job?(digest)
+
+        #       pipeline.zadd(ORPHANED_DIGESTS, now_f, digest)
+        #     end
+        #   end
+        # end
+      end
+
+      def expired_digests
+        conn.zrange(EXPIRING_DIGESTS, 0, max_score, "byscore")
+      end
+
+      def orphaned_digests
+        conn.zrange(ORPHANED_DIGESTS, 0, max_score, "byscore")
+      end
+
+      def max_score
+        (start_time - reaper_timeout - SIDEKIQ_BEAT_PAUSE).to_f
+      end
+
+      #
+      # Find orphaned digests
+      #
+      #
+      # @return [Array<String>] an array of orphaned digests
+      #
+      def orphans # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity
+        orphans = []
+        page    = 0
+        per     = reaper_count * 2
+        results = digests.byscore(0, max_score, offset: page * per, count: (page + 1) * per)
+
+        while results.size.positive?
+          results.each do |digest|
+            break if timeout?
+            next if belongs_to_job?(digest)
+
+            orphans << digest
+            break if orphans.size >= reaper_count
+          end
+
+          break if timeout?
+          break if orphans.size >= reaper_count
+
+          page += 1
+          results = digests.byscore(0, max_score, offset: page * per, count: (page + 1) * per)
+        end
+
+        orphans
+      end
+
+      def timeout?
+        elapsed_ms >= timeout_ms
+      end
+
+      def elapsed_ms
+        time_source.call - start_source
+      end
+
+      #
+      # Checks if the digest has a matching job.
+      #   1. It checks the scheduled set
+      #   2. It checks the retry set
+      #   3. It goes through all queues
+      #
+      #
+      # @param [String] digest the digest to search for
+      #
+      # @return [true] when either of the checks return true
+      # @return [false] when no job was found for this digest
+      #
+      def belongs_to_job?(digest)
+        scheduled?(digest) || retried?(digest) || enqueued?(digest) || active?(digest)
+      end
+
+      #
+      # Checks if the digest exists in the Sidekiq::ScheduledSet
+      #
+      # @param [String] digest the current digest
+      #
+      # @return [true] when digest exists in scheduled set
+      #
+      def scheduled?(digest)
+        in_sorted_set?(SCHEDULE, digest)
+      end
+
+      #
+      # Checks if the digest exists in the Sidekiq::RetrySet
+      #
+      # @param [String] digest the current digest
+      #
+      # @return [true] when digest exists in retry set
+      #
+      def retried?(digest)
+        in_sorted_set?(RETRY, digest)
+      end
+
+      #
+      # Checks if the digest exists in a Sidekiq::Queue
+      #
+      # @param [String] digest the current digest
+      #
+      # @return [true] when digest exists in any queue
+      #
+      def enqueued?(digest)
+        Sidekiq.redis do |conn|
+          queues(conn) do |queue|
+            entries(conn, queue) do |entry|
+              return true if entry.include?(digest)
+            end
+          end
+
+          false
+        end
+      end
+
+      def active?(digest) # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+        Sidekiq.redis do |conn|
+          procs = conn.sscan("processes").to_a
+          return false if procs.empty?
+
+          procs.sort.each do |key|
+            valid, workers = conn.pipelined do |pipeline|
+              # TODO: Remove the if statement in the future
+              if pipeline.respond_to?(:exists?)
+                pipeline.exists?(key)
+              else
+                pipeline.exists(key)
+              end
+              pipeline.hgetall("#{key}:work")
+            end
+
+            next unless valid
+            next unless workers.any?
+
+            workers.each_pair do |_tid, job|
+              next unless (item = safe_load_json(job))
+
+              payload = safe_load_json(item[PAYLOAD])
+
+              return true if match?(digest, payload[LOCK_DIGEST])
+              return true if considered_active?(payload[CREATED_AT])
+            end
+          end
+
+          false
+        end
+      end
+
+      def match?(key_one, key_two)
+        return false if key_one.nil? || key_two.nil?
+
+        key_one.delete_suffix(RUN_SUFFIX) == key_two.delete_suffix(RUN_SUFFIX)
+      end
+
+      def considered_active?(time_f)
+        max_score < time_f
+      end
+
+      #
+      # Loops through all the redis queues and yields them one by one
+      #
+      # @param [Redis] conn the connection to use for fetching queues
+      #
+      # @return [void]
+      #
+      # @yield queues one at a time
+      #
+      def queues(conn, &block)
+        conn.sscan("queues").each(&block)
+      end
+
+      def entries(conn, queue, &block) # rubocop:disable Metrics/MethodLength
+        queue_key    = "queue:#{queue}"
+        initial_size = conn.llen(queue_key)
+        deleted_size = 0
+        page         = 0
+        page_size    = 50
+
+        loop do
+          range_start = (page * page_size) - deleted_size
+
+          range_end   = range_start + page_size - 1
+          entries     = conn.lrange(queue_key, range_start, range_end)
+          page       += 1
+
+          break if entries.empty?
+
+          entries.each(&block)
+
+          deleted_size = initial_size - conn.llen(queue_key)
+
+          # The queue is growing, not shrinking, just keep looping
+          deleted_size = 0 if deleted_size.negative?
+        end
+      end
+
+      # If sidekiq queues are very full, it becomes highly inefficient for the reaper
+      # because it must check every queued job to verify a digest is safe to delete
+      # The reaper checks queued jobs in batches of 50, adding 2 reads per digest
+      # With a queue length of 1,000 jobs, that's over 20 extra reads per digest.
+      def queues_very_full?
+        total_queue_size = 0
+        Sidekiq.redis do |conn|
+          queues(conn) do |queue|
+            total_queue_size += conn.llen("queue:#{queue}")
+
+            return true if total_queue_size > MAX_QUEUE_LENGTH
+          end
+        end
+        false
+      end
+
+      #
+      # Checks a sorted set for the existance of this digest
+      #
+      #
+      # @param [String] key the key for the sorted set
+      # @param [String] digest the digest to scan for
+      #
+      # @return [true] when found
+      # @return [false] when missing
+      #
+      def in_sorted_set?(key, digest)
+        conn.zscan(key, match: "*#{digest}*", count: 1).to_a.any?
+      end
+    end
+    # rubocop:enable Metrics/ClassLength
+  end
+end

data/lib/sidekiq_unique_jobs/redis/entity.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Redis
+    #
+    # Class Entity functions as a base class for redis types
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
+    class Entity
+      # includes "SidekiqUniqueJobs::Logging"
+      # @!parse include SidekiqUniqueJobs::Logging
+      include SidekiqUniqueJobs::Logging
+
+      # includes "SidekiqUniqueJobs::Script::Caller"
+      # @!parse include SidekiqUniqueJobs::Script::Caller
+      include SidekiqUniqueJobs::Script::Caller
+
+      # includes "SidekiqUniqueJobs::JSON"
+      # @!parse include SidekiqUniqueJobs::JSON
+      include SidekiqUniqueJobs::JSON
+
+      # includes "SidekiqUniqueJobs::Timing"
+      # @!parse include SidekiqUniqueJobs::Timing
+      include SidekiqUniqueJobs::Timing
+
+      #
+      # @!attribute [r] key
+      #   @return [String] the redis key for this entity
+      attr_reader :key
+
+      #
+      # Initialize a new Entity
+      #
+      # @param [String] key the redis key for this entity
+      #
+      def initialize(key)
+        @key = key
+      end
+
+      #
+      # Checks if the key for this entity exists in redis
+      #
+      #
+      # @return [true] when exists
+      # @return [false] when not exists
+      #
+      def exist?
+        redis do |conn|
+          # TODO: Remove the if statement in the future
+          value =
+            if conn.respond_to?(:exists?)
+              conn.exists?(key)
+            else
+              conn.exists(key)
+            end
+
+          return value if boolean?(value)
+
+          value.to_i.positive?
+        end
+      end
+
+      #
+      # The number of microseconds until the key expires
+      #
+      #
+      # @return [Integer] expiration in milliseconds
+      #
+      def pttl
+        redis { |conn| conn.pttl(key) }
+      end
+
+      #
+      # The number of seconds until the key expires
+      #
+      #
+      # @return [Integer] expiration in seconds
+      #
+      def ttl
+        redis { |conn| conn.ttl(key) }
+      end
+
+      #
+      # Check if the entity has expiration
+      #
+      #
+      # @return [true] when entity is set to exire
+      # @return [false] when entity isn't expiring
+      #
+      def expires?
+        pttl.positive? || ttl.positive?
+      end
+
+      #
+      # Returns the number of entries in this entity
+      #
+      #
+      # @return [Integer] 0
+      #
+      def count
+        0
+      end
+
+      private
+
+      def boolean?(value)
+        [TrueClass, FalseClass].any? { |klazz| value.is_a?(klazz) }
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/redis/hash.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Redis
+    #
+    # Class Hash provides convenient access to redis hashes
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
+    class Hash < Entity
+      #
+      # Return entries for this hash
+      #
+      # @param [true,false] with_values false return hash
+      #
+      # @return [Array<Object>] when given with_values: false
+      # @return [Hash<String, String>] when given with_values: true
+      #
+      def entries(with_values: false)
+        if with_values
+          redis { |conn| conn.hgetall(key) }
+        else
+          redis { |conn| conn.hkeys(key) }
+        end
+      end
+
+      #
+      # Removes the key from redis
+      #
+      def del(*fields)
+        redis { |conn| conn.hdel(key, *fields) }
+      end
+
+      #
+      # Get a members value
+      #
+      # @param [String] member the member who's value to get
+      #
+      # @return [Object] whatever is stored on this hash member
+      #
+      def [](member)
+        redis { |conn| conn.hget(key, member) }
+      end
+
+      #
+      # Returns the count for this hash
+      #
+      #
+      # @return [Integer] the length of this hash
+      #
+      def count
+        redis { |conn| conn.hlen(key) }
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/redis/list.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Redis
+    #
+    # Class List provides convenient access to redis hashes
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
+    class List < Entity
+      #
+      # Entries in this list
+      #
+      #
+      # @return [Array<Object>] the elements in this list
+      #
+      def entries
+        redis { |conn| conn.lrange(key, 0, -1) }
+      end
+
+      #
+      # The number of entries in this list
+      #
+      #
+      # @return [Integer] the total number of entries
+      #
+      def count
+        redis { |conn| conn.llen(key) }
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/redis/set.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Redis
+    #
+    # Class Set provides convenient access to redis sets
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
+    class Set < Entity
+      #
+      # Return entries for this set
+      #
+      #
+      # @return [Array<String>]
+      #
+      def entries
+        redis { |conn| conn.smembers(key) }
+      end
+
+      #
+      # Returns the count for this sorted set
+      #
+      #
+      # @return [Integer] the number of entries
+      #
+      def count
+        redis { |conn| conn.scard(key) }
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/redis/sorted_set.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Redis
+    #
+    # Class SortedSet provides convenient access to redis sorted sets
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
+    class SortedSet < Entity
+      #
+      # @return [Integer] the number of matches to return by default
+      DEFAULT_COUNT = 1_000
+      #
+      # @return [String] the default pattern to use for matching
+      SCAN_PATTERN  = "*"
+
+      #
+      # Return entries for this sorted set
+      #
+      # @param [true,false] with_scores true return
+      #
+      # @return [Array<Object>] when given with_scores: false
+      # @return [Hash] when given with_scores: true
+      #
+      def entries(with_scores: true)
+        return redis { |conn| conn.zrange(key, 0, -1) } unless with_scores
+
+        redis { |conn| conn.zrange(key, 0, -1, "withscores") }.each_with_object({}) do |pair, hash|
+          hash[pair[0]] = pair[1]
+        end
+      end
+
+      #
+      # Adds a value to the sorted set
+      #
+      # @param [Array<Float, String>, String] values the values to add
+      #
+      # @return [Boolean, Integer] <description>
+      #
+      def add(values)
+        redis do |conn|
+          if values.is_a?(Array)
+            conn.zadd(key, *values)
+          else
+            conn.zadd(key, now_f, values)
+          end
+        end
+      end
+
+      def byscore(min, max, offset: nil, count: nil)
+        redis do |conn|
+          return conn.zrange(key, min, max, "byscore") unless offset && count
+
+          conn.zrange(key, min, max, "byscore", "limit", offset, count)
+        end
+      end
+
+      #
+      # Return the zrak of the member
+      #
+      # @param [String] member the member to pull rank on
+      #
+      # @return [Integer]
+      #
+      def rank(member)
+        redis { |conn| conn.zrank(key, member) }
+      end
+
+      #
+      # Return score for a member
+      #
+      # @param [String] member the member for which score
+      #
+      # @return [Integer, Float]
+      #
+      def score(member)
+        redis { |conn| conn.zscore(key, member) }
+      end
+
+      #
+      # Clears the sorted set from all entries
+      #
+      #
+      # @return [Integer] number of entries removed
+      #
+      def clear
+        redis { |conn| conn.zremrangebyrank(key, 0, count) }
+      end
+
+      #
+      # Returns the count for this sorted set
+      #
+      #
+      # @return [Integer] the number of entries
+      #
+      def count
+        redis { |conn| conn.zcard(key) }
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/redis/string.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  module Redis
+    #
+    # Class String provides convenient access to redis strings
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
+    class String < Entity
+      #
+      # Returns the value of the key
+      #
+      #
+      # @return [String]
+      #
+      def value
+        redis { |conn| conn.get(key) }
+      end
+
+      #
+      # Sets the value of the key to given object
+      #
+      # @param [String] obj the object to update the key with
+      #
+      # @return [true, false]
+      #
+      def set(obj, pipeline = nil)
+        return pipeline.set(key, obj) if pipeline
+
+        redis { |conn| conn.set(key, obj) }
+      end
+
+      #
+      # Removes the key from redis
+      #
+      def del(*)
+        redis { |conn| conn.del(key) }
+      end
+
+      #
+      # Used only for compatibility with other keys
+      #
+      # @return [1] when key exists
+      # @return [0] when key does not exists
+      def count
+        exist? ? 1 : 0
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/redis.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Augments and enhances redis with object oriented methods
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  module Redis
+  end
+end