sidekiq-ultimate 0.0.1.alpha.19 → 2.0.0

data/lib/sidekiq/ultimate/fetch.rb

@@ -1,11 +1,14 @@
 # frozen_string_literal: true
 
+require "sidekiq"
 require "sidekiq/throttled"
 
 require "sidekiq/ultimate/expirable_set"
 require "sidekiq/ultimate/queue_name"
 require "sidekiq/ultimate/resurrector"
 require "sidekiq/ultimate/unit_of_work"
+require "sidekiq/ultimate/empty_queues"
+require "sidekiq/ultimate/configuration"
 
 module Sidekiq
   module Ultimate
@@ -14,17 +17,11 @@ module Sidekiq
       # Delay between fetch retries in case of no job received.
       TIMEOUT = 2
 
-      # Delay between queue poll attempts if last poll returned no jobs for it.
-      QUEUE_TIMEOUT = 5
-
-      # Delay between queue poll attempts if it's last job was throttled.
-      THROTTLE_TIMEOUT = 15
-
       def initialize(options)
-        @exhausted = ExpirableSet.new
-
+        @exhausted_by_throttling = ExpirableSet.new
+        @empty_queues = Sidekiq::Ultimate::EmptyQueues.instance
         @strict = options[:strict] ? true : false
-        @queues = options[:queues].map { |name| QueueName.new(name) }
+        @queues = options[:queues]
 
         @queues.uniq! if @strict
 
@@ -39,8 +36,9 @@ module Sidekiq
         if work&.throttled?
           work.requeue_throttled
 
-          queue = QueueName.new(work.queue_name)
-          @exhausted.add(queue, :ttl => THROTTLE_TIMEOUT)
+          @exhausted_by_throttling.add(
+            work.queue_name, :ttl => Sidekiq::Ultimate::Configuration.instance.throttled_fetch_timeout_sec
+          )
 
           return nil
         end
@@ -48,24 +46,26 @@ module Sidekiq
         work
       end
 
-      def self.bulk_requeue(units, _options)
+      # TODO: Requeue in batch or at least using pipeline
+      def bulk_requeue(units, _options)
         units.each(&:requeue)
       end
 
       def self.setup!
-        Sidekiq.options[:fetch] = self
+        fetcher = new(Sidekiq)
+
+        Sidekiq[:fetch] = fetcher
         Resurrector.setup!
+        EmptyQueues.setup!
       end
 
       private
 
       def retrieve
         Sidekiq.redis do |redis|
-          queues.each do |queue|
+          queues_objects.each do |queue|
             job = redis.rpoplpush(queue.pending, queue.inproc)
             return UnitOfWork.new(queue, job) if job
-
-            @exhausted.add(queue, :ttl => QUEUE_TIMEOUT)
           end
         end
 
@@ -73,19 +73,19 @@ module Sidekiq
         nil
       end
 
-      def queues
-        queues = (@strict ? @queues : @queues.shuffle.uniq) - @exhausted.to_a
+      def queues_objects
+        queues = (@strict ? @queues : @queues.shuffle.uniq) - @exhausted_by_throttling.to_a - @empty_queues.queues
 
         # Avoid calling heavier `paused_queue` if there's nothing to filter out
-        return queues if queues.empty?
+        return [] if queues.empty?
 
-        queues - paused_queues
+        (queues - paused_queues).map { |name| QueueName.new(name) }
       end
 
       def paused_queues
         return @paused_queues if Time.now.to_i < @paused_queues_expires_at
 
-        @paused_queues.replace(Sidekiq::Throttled::QueuesPauser.instance.paused_queues.map { |q| QueueName[q] })
+        @paused_queues = Sidekiq::Throttled::QueuesPauser.instance.paused_queues
         @paused_queues_expires_at = Time.now.to_i + 60
 
         @paused_queues

data/lib/sidekiq/ultimate/interval_with_jitter.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Ultimate
+    # Util class to add a jitter to the interval
+    class IntervalWithJitter
+      RANDOM_OFFSET_RATIO = 0.1
+
+      class << self
+        # Returns execution interval with jitter.
+        # Jitter is +- RANDOM_OFFSET_RATIO from the original value.
+        def call(interval)
+          jitter_factor = 1 + rand(-RANDOM_OFFSET_RATIO..RANDOM_OFFSET_RATIO)
+          jitter_factor * interval
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/ultimate/queue_name.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "sidekiq/util"
+require "sidekiq/component"
 
 module Sidekiq
   module Ultimate
@@ -10,11 +10,11 @@ module Sidekiq
     class QueueName
       # Regexp used to normalize (possibly) expanded queue name, e.g. the one
       # that is returned upon redis BRPOP
-      QUEUE_PREFIX_RE = %r{.*queue:}
+      QUEUE_PREFIX_RE = %r{.*queue:}.freeze
       private_constant :QUEUE_PREFIX_RE
 
       # Internal helper context.
-      Helper = Module.new { extend Sidekiq::Util }
+      Helper = Module.new { extend Sidekiq::Component }
       private_constant :Helper
 
       # Original stringified queue name.
@@ -29,8 +29,7 @@ module Sidekiq
 
       # Create a new QueueName instance.
       #
-      # @param normalized [#to_s] Normalized (without any namespaces or `queue:`
-      #   prefixes) queue name.
+      # @param normalized [#to_s] Normalized (without `queue:` prefixes) queue name.
       # @param identity [#to_s] Sidekiq process identity.
       def initialize(normalized, identity: Helper.identity)
         @normalized = -normalized.to_s

data/lib/sidekiq/ultimate/resurrector/common_constants.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Ultimate
+    module Resurrector
+      module CommonConstants
+        MAIN_KEY = "ultimate:resurrector"
+
+        RESURRECTOR_INTERVAL = 60
+      end
+    end
+  end
+end

data/lib/sidekiq/ultimate/resurrector/count.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "sidekiq/ultimate/resurrector/common_constants"
+
+module Sidekiq
+  module Ultimate
+    module Resurrector
+      # Allows to get the count of times the job was resurrected
+      module Count
+        class << self
+          # @param job_id [String] job id
+          # @return [Integer] count of times the job was resurrected
+          def read(job_id:)
+            Sidekiq.redis do |redis|
+              redis.get("#{CommonConstants::MAIN_KEY}:counter:jid:#{job_id}").to_i
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/ultimate/resurrector/lock.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "redlock"
+require "sidekiq/ultimate/resurrector/common_constants"
+
+module Sidekiq
+  module Ultimate
+    module Resurrector
+      # Ensures exclusive access to resurrection process
+      class Lock
+        LOCK_KEY = "#{CommonConstants::MAIN_KEY}:lock"
+        private_constant :LOCK_KEY
+
+        LAST_RUN_KEY = "#{CommonConstants::MAIN_KEY}:last_run"
+        private_constant :LAST_RUN_KEY
+
+        LOCK_TTL = 30_000 # ms
+        private_constant :LOCK_TTL
+
+        class << self
+          def acquire
+            Sidekiq.redis do |redis|
+              break if resurrected_recently?(redis) # Cheap check since lock will not be free most of the time
+
+              Redlock::Client.new([redis], :retry_count => 0).lock(LOCK_KEY, LOCK_TTL) do |locked|
+                break unless locked
+                break if resurrected_recently?(redis)
+
+                yield
+
+                redis.set(LAST_RUN_KEY, redis.time.first)
+              end
+            end
+          end
+
+          def resurrected_recently?(redis)
+            results  = redis.pipelined { |pipeline| [pipeline.time, pipeline.get(LAST_RUN_KEY)] }
+            distance = results[0][0] - results[1].to_i
+
+            distance < CommonConstants::RESURRECTOR_INTERVAL
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/ultimate/resurrector/lua_scripts/resurrect_with_counter.lua

@@ -0,0 +1,22 @@
+local resurrected_jobs = 0
+
+while true do
+  local job_data = redis.call("LPOP", KEYS[1])
+
+  if job_data then
+    redis.call("RPUSH", KEYS[2], job_data)
+
+    resurrected_jobs = resurrected_jobs + 1
+
+    local _, jid_position = string.find(job_data, "\"jid\"")
+    jid_position = jid_position + 3
+
+    local jid = job_data:sub(jid_position, jid_position + 23)
+    local jid_key =  KEYS[3] .. ':counter:jid:' .. jid
+
+    redis.call("INCR", jid_key)
+    redis.call("EXPIRE", jid_key, 86400)
+  else
+    return resurrected_jobs
+  end
+end

data/lib/sidekiq/ultimate/resurrector/resurrection_script.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require "redis_prescription"
+require "sidekiq/ultimate/configuration"
+require "sidekiq/ultimate/resurrector/common_constants"
+
+module Sidekiq
+  module Ultimate
+    module Resurrector
+      # Lost jobs checker and resurrector
+      class ResurrectionScript
+        RESURRECT = RedisPrescription.new(File.read("#{__dir__}/lua_scripts/resurrect.lua"))
+        private_constant :RESURRECT
+
+        RESURRECT_WITH_COUNTER = RedisPrescription.new(File.read("#{__dir__}/lua_scripts/resurrect_with_counter.lua"))
+        private_constant :RESURRECT_WITH_COUNTER
+
+        def self.call(*args)
+          new.call(*args)
+        end
+
+        def call(redis, keys:)
+          # redis-namespace can only namespace arguments of the lua script, so we need to pass the main key
+          keys += [CommonConstants::MAIN_KEY] if enable_resurrection_counter
+          script.call(redis, :keys => keys)
+        end
+
+        private
+
+        def script
+          enable_resurrection_counter ? RESURRECT_WITH_COUNTER : RESURRECT
+        end
+
+        def enable_resurrection_counter
+          return @enable_resurrection_counter if defined?(@enable_resurrection_counter)
+
+          @enable_resurrection_counter =
+            if enable_resurrection_counter_setting.respond_to?(:call)
+              enable_resurrection_counter_setting.call
+            else
+              enable_resurrection_counter_setting
+            end
+        end
+
+        def enable_resurrection_counter_setting
+          Sidekiq::Ultimate::Configuration.instance.enable_resurrection_counter
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/ultimate/resurrector.rb

@@ -1,41 +1,39 @@
 # frozen_string_literal: true
 
-require "redis/lockers"
-require "redis/prescription"
+require "redis_prescription"
+require "concurrent/timer_task"
 
+require "sidekiq/component"
 require "sidekiq/ultimate/queue_name"
+require "sidekiq/ultimate/resurrector/lock"
+require "sidekiq/ultimate/resurrector/common_constants"
+require "sidekiq/ultimate/resurrector/resurrection_script"
+require "sidekiq/ultimate/configuration"
+require "sidekiq/ultimate/interval_with_jitter"
 
 module Sidekiq
   module Ultimate
-    # Lost jobs resurrector.
+    # Lost jobs checker and resurrector
     module Resurrector
-      RESURRECT = Redis::Prescription.read \
-        "#{__dir__}/resurrector/resurrect.lua"
-      private_constant :RESURRECT
-
-      SAFECLEAN = Redis::Prescription.read \
-        "#{__dir__}/resurrector/safeclean.lua"
+      SAFECLEAN = RedisPrescription.new(File.read("#{__dir__}/resurrector/lua_scripts/safeclean.lua"))
       private_constant :SAFECLEAN
 
-      MAIN_KEY = "ultimate:resurrector"
-      private_constant :MAIN_KEY
-
-      LOCK_KEY = "#{MAIN_KEY}:lock"
-      private_constant :LOCK_KEY
+      DEFIBRILLATE_INTERVAL = 5
+      private_constant :DEFIBRILLATE_INTERVAL
 
-      LAST_RUN_KEY = "#{MAIN_KEY}:last_run"
-      private_constant :LAST_RUN_KEY
+      ResurrectorTimerTask = Class.new(Concurrent::TimerTask)
+      HeartbeatTimerTask = Class.new(Concurrent::TimerTask)
 
       class << self
         def setup!
-          @identity = Object.new.tap { |o| o.extend Sidekiq::Util }.identity
-
-          register_aed!
-          call_cthulhu!
+          register_process_heartbeat
+          register_resurrector
         end
 
+        # go over all sidekiq processes (identities) that were shut down recently, get all their queues and
+        # try to resurrect them
         def resurrect!
-          lock do
+          Sidekiq::Ultimate::Resurrector::Lock.acquire do
             casualties.each do |identity|
               log(:debug) { "Resurrecting #{identity}" }
 
@@ -48,102 +46,97 @@ module Sidekiq
           raise
         end
 
+        def current_process_identity
+          @current_process_identity ||= Object.new.tap { |o| o.extend Sidekiq::Component }.identity
+        end
+
         private
 
-        def call_cthulhu!
-          cthulhu = nil
+        def register_resurrector
+          resurrector_timer_task = nil
 
           Sidekiq.on(:startup) do
-            cthulhu&.shutdown
+            resurrector_timer_task&.shutdown
 
-            cthulhu = Concurrent::TimerTask.execute({
+            resurrector_timer_task = ResurrectorTimerTask.new({
               :run_now            => true,
-              :execution_interval => 60
+              :execution_interval => Sidekiq::Ultimate::IntervalWithJitter.call(CommonConstants::RESURRECTOR_INTERVAL)
             }) { resurrect! }
+            resurrector_timer_task.execute
           end
 
-          Sidekiq.on(:shutdown) { cthulhu&.shutdown }
+          Sidekiq.on(:shutdown) { resurrector_timer_task&.shutdown }
         end
 
-        def register_aed!
-          aed = nil
+        def register_process_heartbeat
+          heartbeat_timer_task = nil
 
           Sidekiq.on(:heartbeat) do
-            aed&.shutdown
+            heartbeat_timer_task&.shutdown
 
-            aed = Concurrent::TimerTask.execute({
+            heartbeat_timer_task = HeartbeatTimerTask.new({
               :run_now            => true,
-              :execution_interval => 5
-            }) { defibrillate! }
+              :execution_interval => Sidekiq::Ultimate::IntervalWithJitter.call(DEFIBRILLATE_INTERVAL)
+            }) { save_watched_queues }
+            heartbeat_timer_task.execute
           end
 
-          Sidekiq.on(:shutdown) { aed&.shutdown }
+          Sidekiq.on(:shutdown) { heartbeat_timer_task&.shutdown }
         end
 
-        def defibrillate!
+        # put current list of queues into resurrection candidates
+        def save_watched_queues
           Sidekiq.redis do |redis|
             log(:debug) { "Defibrillating" }
 
-            queues = JSON.dump(Sidekiq.options[:queues].uniq)
-            redis.hset(MAIN_KEY, @identity, queues)
-          end
-        end
-
-        def lock
-          Sidekiq.redis do |redis|
-            Redis::Lockers.acquire(redis, LOCK_KEY, :ttl => 30_000) do
-              results  = redis.pipelined { |r| [r.time, r.get(LAST_RUN_KEY)] }
-              distance = results[0][0] - results[1].to_i
-
-              break unless 60 < distance
-
-              yield
-
-              redis.set(LAST_RUN_KEY, redis.time.first)
-            end
+            queues = JSON.dump(Sidekiq[:queues].uniq)
+            redis.hset(CommonConstants::MAIN_KEY, current_process_identity, queues)
           end
         end
 
+        # list of processes that disappeared after latest #save_watched_queues
         def casualties
           Sidekiq.redis do |redis|
-            casualties = []
-            identities = redis.hkeys(MAIN_KEY)
+            sidekiq_processes = redis.hkeys(CommonConstants::MAIN_KEY)
 
-            redis.pipelined { identities.each { |k| redis.exists k } }.
-              each_with_index { |v, i| casualties << identities[i] unless v }
+            sidekiq_processes_alive = redis.pipelined do |pipeline|
+              sidekiq_processes.each do |process|
+                pipeline.exists?(process)
+              end
+            end
 
-            casualties
+            sidekiq_processes.zip(sidekiq_processes_alive).reject { |(_, alive)| alive }.map(&:first)
           end
         end
 
+        # Get list of genuine sidekiq queues names for a given identity (sidekiq process id)
         def queues_of(identity)
           Sidekiq.redis do |redis|
-            queues = redis.hget(MAIN_KEY, identity)
+            queues = redis.hget(CommonConstants::MAIN_KEY, identity)
 
             return [] unless queues
 
-            JSON.parse(queues).map do |q|
-              QueueName.new(q, :identity => identity)
-            end
+            JSON.parse(queues).map { |q| QueueName.new(q, :identity => identity) }
           end
         end
 
+        # Move jobs from inproc to pending
         def resurrect(queue)
           Sidekiq.redis do |redis|
-            result = RESURRECT.eval(redis, {
-              :keys => [queue.inproc, queue.pending]
-            })
+            resurrected_jobs_count = ResurrectionScript.call(redis, :keys => [queue.inproc, queue.pending])
 
-            if result.positive?
-              log(:info) { "Resurrected #{result} jobs from #{queue.inproc}" }
+            if resurrected_jobs_count.positive?
+              log(:info) { "Resurrected #{resurrected_jobs_count} jobs from #{queue.inproc}" }
+              Sidekiq::Ultimate::Configuration.instance.on_resurrection&.call(queue.to_s, resurrected_jobs_count)
             end
           end
         end
 
+        # Delete empty inproc queues and clean up identity key from resurrection candidates (CommonConstants::MAIN_KEY)
         def cleanup(identity, inprocs)
           Sidekiq.redis do |redis|
-            result = SAFECLEAN.eval(redis, {
-              :keys => [MAIN_KEY, *inprocs],
+            result = SAFECLEAN.call(redis, {
+              :keys => [CommonConstants::MAIN_KEY, *inprocs],
               :argv => [identity]
             })
 
@@ -153,7 +146,7 @@ module Sidekiq
 
         def log(level)
           Sidekiq.logger.public_send(level) do
-            "[#{self}] @#{@identity} #{yield}"
+            "[#{self}] @#{current_process_identity} #{yield}"
           end
         end
       end

data/lib/sidekiq/ultimate/unit_of_work.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "redis/prescription"
+require "redis_prescription"
 
 require "sidekiq/throttled"
 
@@ -10,8 +10,7 @@ module Sidekiq
     #
     # @private
     class UnitOfWork
-      REQUEUE = Redis::Prescription.read \
-        "#{__dir__}/unit_of_work/requeue.lua"
+      REQUEUE = RedisPrescription.new(File.read("#{__dir__}/unit_of_work/requeue.lua"))
       private_constant :REQUEUE
 
       # JSON payload
@@ -90,12 +89,14 @@ module Sidekiq
 
       private
 
+      # If the jobs was in the inproc queue, then delete it from there and
+      # push the job back to the queue using `command`.
       def __requeue__(command)
         @mutex.synchronize do
           return if @requeued || @acked
 
           Sidekiq.redis do |redis|
-            REQUEUE.eval(redis, {
+            REQUEUE.call(redis, {
               :keys => [@queue.pending, @queue.inproc],
               :argv => [command, @job]
             })

data/lib/sidekiq/ultimate/version.rb

@@ -2,7 +2,6 @@
 
 module Sidekiq
   module Ultimate
-    # Gem version.
-    VERSION = "0.0.1.alpha.19"
+    VERSION = "2.0.0"
   end
 end

data/lib/sidekiq/ultimate.rb

@@ -2,7 +2,8 @@
 
 require "sidekiq/throttled"
 
-require "sidekiq/ultimate/version"
+require "sidekiq/ultimate/configuration"
+require "sidekiq/ultimate/resurrector/count"
 
 module Sidekiq
   # Sidekiq ultimate experience.
@@ -10,10 +11,18 @@ module Sidekiq
     class << self
       # Sets up reliable throttled fetch and friends.
       # @return [void]
-      def setup!
+      def setup!(&configuration_block)
+        configuration_block&.call(Sidekiq::Ultimate::Configuration.instance)
+
         Sidekiq::Throttled::Communicator.instance.setup!
         Sidekiq::Throttled::QueuesPauser.instance.setup!
 
+        sidekiq_configure_server
+      end
+
+      private
+
+      def sidekiq_configure_server
         Sidekiq.configure_server do |config|
           require "sidekiq/ultimate/fetch"
           Sidekiq::Ultimate::Fetch.setup!

data/sidekiq-ultimate.gemspec

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 require "sidekiq/ultimate/version"
 
-Gem::Specification.new do |spec|
+Gem::Specification.new do |spec| # rubocop:disable Gemspec/RequireMFA
   spec.name          = "sidekiq-ultimate"
   spec.version       = Sidekiq::Ultimate::VERSION
   spec.authors       = ["Alexey Zapparov"]
@@ -23,15 +23,13 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_runtime_dependency "concurrent-ruby",    "~> 1.0"
-  spec.add_runtime_dependency "redis-lockers",      "~> 1.1"
-  spec.add_runtime_dependency "redis-prescription", "~> 1.0"
-  spec.add_runtime_dependency "sidekiq",            "~> 5.0"
-
-  # temporary couple this with sidekiq-throttled until it will be merged into
-  # this gem instead.
-  spec.add_runtime_dependency "sidekiq-throttled",  "~> 0.8"
+  spec.add_runtime_dependency "redis",              "~> 4.8"
+  spec.add_runtime_dependency "redis-prescription", "~> 2.6"
+  spec.add_runtime_dependency "redlock",            "~> 1.3"
+  spec.add_runtime_dependency "sidekiq",            "~> 6.5.0"
+  spec.add_runtime_dependency "sidekiq-throttled",  "~> 0.18.0"
 
   spec.add_development_dependency "bundler", "~> 2.0"
 
-  spec.required_ruby_version = "~> 2.3"
+  spec.required_ruby_version = "~> 2.7"
 end