sidekiq-priority_queue 1.0.2 → 1.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 23819a77a9986da48fe0e15a590bd948abf62cafa7d48f7ad8aa2d6129dbb476
-  data.tar.gz: cc8e7a8a99c095237a6b44c0a266753d98707c724e5af70b494a71b3a25476ac
+  metadata.gz: 0e649f9936ca3989853559d9254ce6072229a700e5b5d4f17efc1dbf723c0cbe
+  data.tar.gz: d53da6399ee5f1405598de900b753ebbe3cb3b7b45e7ca2991b915c76ac1a3a8
 SHA512:
-  metadata.gz: '0884b842b53184df96c7e753d0d8d6b04621b2b5810a33e5798e95d76b13e09776ac4db62c520c0e822be0987b798deaa473e76898ecdd92b4095230b6b65501'
-  data.tar.gz: f1bb0dfb7a3048ce7ca80924916ac6817ed16262a2d98596f1da50db35292b2a2cb3287f0fcdb61a468f21f39d0dac5de3f8989b7bdc87515f728179fbb7576a
+  metadata.gz: 9cf8bc3aca671561f792ce90592296655dba58541e0506c362c73ed3c54b42a8978235423c327f714b6f4749f9585ad29a942547ac5378621cfa1441bb215ef2
+  data.tar.gz: 8be1d8ad3d836104c19bc3e2473b6d77459ea3938557bd4672e72b92b3dc10a0e1cb2c2d94bc576e3c346c49a27528f94024bd9613603ef2e22de5ff1343d4ba

data/Gemfile.lock

@@ -1,15 +1,15 @@
 PATH
   remote: .
   specs:
-    sidekiq-priority_queue (1.0.2)
-      sidekiq (>= 6)
+    sidekiq-priority_queue (1.0.5)
+      sidekiq (>= 6.2.2)
 
 GEM
   remote: https://rubygems.org/
   specs:
     byebug (11.1.3)
     coderay (1.1.3)
-    connection_pool (2.2.3)
+    connection_pool (2.2.5)
     docile (1.3.4)
     method_source (1.0.0)
     minitest (5.14.3)
@@ -23,8 +23,8 @@ GEM
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
     rake (13.0.3)
-    redis (4.2.5)
-    sidekiq (6.1.3)
+    redis (4.5.1)
+    sidekiq (6.2.2)
       connection_pool (>= 2.2.2)
       rack (~> 2.0)
       redis (>= 4.2.0)
@@ -47,4 +47,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.1.4
+   2.2.3

data/lib/sidekiq/priority_queue/api.rb

@@ -50,7 +50,7 @@ module Sidekiq
 
     SubqueueCount = Struct.new(:name, :size)
 
-    class Job < Sidekiq::Job
+    class Job < Sidekiq::JobRecord
 
       attr_reader :priority
       attr_reader :subqueue

data/lib/sidekiq/priority_queue/reliable_fetch.rb

@@ -1,9 +1,14 @@
 # frozen_string_literal: true
+
 require 'sidekiq'
+require 'sidekiq/util'
 
 module Sidekiq
   module PriorityQueue
     class ReliableFetch
+      include Sidekiq::Util
+
+      SUPER_PROCESSES_REGISTRY_KEY = 'super_processes_priority'
 
       UnitOfWork = Struct.new(:queue, :job, :wip_queue) do
         def acknowledge
@@ -35,33 +40,51 @@ module Sidekiq
       end
 
       def initialize(options)
+        @options = options
         @strictly_ordered_queues = !!options[:strict]
         @queues = options[:queues].map { |q| "priority-queue:#{q}" }
         @queues = @queues.uniq if @strictly_ordered_queues
+        @done = false
         @process_index = options[:index] || ENV['PROCESS_INDEX']
       end
 
+      def setup
+        Sidekiq.on(:startup) do
+          cleanup_the_dead
+          register_myself
+          check
+        end
+        Sidekiq.on(:shutdown) do
+          @done = true
+        end
+        Sidekiq.on(:heartbeat) do
+          register_myself
+        end
+      end
+
       def retrieve_work
+        return nil if @done
+
         work = @queues.detect do |q|
-          job = zpopmin_sadd(q, wip_queue(q));
-          break [q,job] if job
+          job = zpopmin_sadd(q, wip_queue(q))
+          break [q, job] if job
         end
         UnitOfWork.new(*work, wip_queue(work.first)) if work
       end
 
       def wip_queue(q)
-        "#{q}_#{Socket.gethostname}_#{@process_index}"
+        "queue:spriorityq|#{identity}|#{q}"
       end
 
       def zpopmin_sadd(queue, wip_queue)
-        Sidekiq.redis do |con|
-          @script_sha ||= con.script(:load, Sidekiq::PriorityQueue::Scripts::ZPOPMIN_SADD)
-          con.evalsha(@script_sha, [queue, wip_queue])
+        Sidekiq.redis do |conn|
+          @script_sha ||= conn.script(:load, Sidekiq::PriorityQueue::Scripts::ZPOPMIN_SADD)
+          conn.evalsha(@script_sha, [queue, wip_queue])
         end
       end
 
       def spop(wip_queue)
-        Sidekiq.redis{ |con| con.spop(wip_queue) }
+        Sidekiq.redis { |con| con.spop(wip_queue) }
       end
 
       def queues_cmd
@@ -72,55 +95,169 @@ module Sidekiq
         end
       end
 
-      def bulk_requeue(_inprogress, options)
-        Sidekiq.logger.debug { "Re-queueing terminated jobs" }
-        process_index = options[:index] || ENV['PROCESS_INDEX']
-        self.class.requeue_wip_jobs(options[:queues], process_index)
+      # Below method is called when we close sidekiq process gracefully
+      def bulk_requeue(_inprogress, _options)
+        Sidekiq.logger.debug { 'Priority ReliableFetch: Re-queueing terminated jobs' }
+        requeue_wip_jobs
+        unregister_super_process
+      end
+
+      private
+
+      def check
+        check_for_orphans if orphan_check?
+      rescue StandardError => e
+        # orphan check is best effort, we don't want Redis downtime to
+        # break Sidekiq
+        Sidekiq.logger.warn { "Priority ReliableFetch: Failed to do orphan check: #{e.message}" }
       end
 
-      def self.resume_wip_jobs(queues, process_index)
-        Sidekiq.logger.debug { "Re-queueing WIP jobs" }
-        process_index ||= ENV['PROCESS_INDEX']
-        requeue_wip_jobs(queues, process_index)
+      def orphan_check?
+        delay = @options.fetch(:reliable_fetch_orphan_check, 3600).to_i
+        return false if delay.zero?
+
+        Sidekiq.redis do |conn|
+          conn.set('priority_reliable_fetch_orphan_check', Time.now.to_f, ex: delay, nx: true)
+        end
       end
 
-      Sidekiq.configure_server do |config|
-        config.on(:startup) do
-          if reliable_fetch_active?(config)
-            Sidekiq::PriorityQueue::ReliableFetch.resume_wip_jobs(config.options[:queues], config.options[:index])
+      # This method is extra paranoid verification to check Redis for any possible
+      # orphaned queues with jobs. If we change queue names and lose jobs in the meantime,
+      # this will find old queues with jobs and rescue them.
+      def check_for_orphans
+        orphans_count = 0
+        queues_count = 0
+        orphan_queues = Set.new
+        Sidekiq.redis do |conn|
+          ids = conn.smembers(SUPER_PROCESSES_REGISTRY_KEY)
+          Sidekiq.logger.debug("Priority ReliableFetch found #{ids.size} super processes")
+
+          conn.scan_each(match: 'queue:spriorityq|*', count: 100) do |wip_queue|
+            queues_count += 1
+            _, id, original_priority_queue_name = wip_queue.split('|')
+            next if ids.include?(id)
+
+            # Race condition in pulling super_processes and checking queue liveness.
+            # Need to verify in Redis.
+            unless conn.sismember(SUPER_PROCESSES_REGISTRY_KEY, id)
+              orphan_queues << original_priority_queue_name
+              queue_jobs_count = 0
+              loop do
+                break if conn.scard(wip_queue).zero?
+
+                # Here we should wrap below two operations in Lua script
+                item = conn.spop(wip_queue)
+                conn.zadd(original_priority_queue_name, 0, item)
+                orphans_count += 1
+                queue_jobs_count += 1
+              end
+              if queue_jobs_count.positive?
+                Sidekiq::Pro.metrics.increment('jobs.recovered.fetch', by: queue_jobs_count, tags: ["queue:#{original_priority_queue_name}"])
+              end
+            end
           end
         end
+
+        if orphans_count.positive?
+          Sidekiq.logger.warn { "Priority ReliableFetch recovered #{orphans_count} orphaned jobs in queues: #{orphan_queues.to_a.inspect}" }
+        elsif queues_count.positive?
+          Sidekiq.logger.info { "Priority ReliableFetch found #{queues_count} working queues with no orphaned jobs" }
+        end
+        orphans_count
       end
 
-      private
+      # Below method is only to make sure we get jobs from incorrectly closed process (for example force killed using kill -9 SIDEKIQ_PID)
+      def cleanup_the_dead
+        overall_moved_count = 0
+        Sidekiq.redis do |conn|
+          conn.sscan_each(SUPER_PROCESSES_REGISTRY_KEY) do |super_process|
+            next if conn.exists?(super_process) # Don't clean up currently running processes
+
+            Sidekiq.logger.debug { "Priority ReliableFetch: Moving job from #{super_process} back to original queues" }
+
+            # We need to pushback any leftover jobs still in WIP
+            previously_handled_queues = conn.smembers("#{super_process}:super_priority_queues")
+
+            # Below previously_handled_queues are simply WIP queues of previous, dead processes
+            previously_handled_queues.each do |previously_handled_queue|
+              queue_moved_size = 0
+              original_priority_queue_name = previously_handled_queue.split('|').last
+
+              Sidekiq.logger.debug { "Priority ReliableFetch: Moving job from #{previously_handled_queue} back to original queue: #{original_priority_queue_name}" }
+              loop do
+                break if conn.scard(previously_handled_queue).zero?
+
+                # Here we should wrap below two operations in Lua script
+                item = conn.spop(previously_handled_queue)
+                conn.zadd(original_priority_queue_name, 0, item)
+                queue_moved_size += 1
+                overall_moved_count += 1
+              end
+              # Below we simply remove old WIP queue
+              conn.del(previously_handled_queue) if conn.scard(previously_handled_queue).zero?
+              Sidekiq.logger.debug { "Priority ReliableFetch: Moved #{queue_moved_size} jobs from ##{previously_handled_queue} back to original_queue: #{original_priority_queue_name} " }
+            end
 
-      def self.reliable_fetch_active?(config)
-        return true if config.options[:fetch].is_a?(Sidekiq::PriorityQueue::ReliableFetch)
-        return config.options[:fetch].is_a?(Sidekiq::PriorityQueue::CombinedFetch) &&
-          config.options[:fetch].fetches.any? { |f| f.is_a?(Sidekiq::PriorityQueue::ReliableFetch) }
+            Sidekiq.logger.debug { "Priority ReliableFetch: Unregistering super process #{super_process}" }
+            conn.del("#{super_process}:super_priority_queues")
+            conn.srem(SUPER_PROCESSES_REGISTRY_KEY, super_process)
+          end
+        end
+        Sidekiq.logger.debug { "Priority ReliableFetch: Moved overall #{overall_moved_count} jobs from WIP queues" }
+      rescue StandardError => e
+        # best effort, ignore Redis network errors
+        Sidekiq.logger.warn { "Priority ReliableFetch: Failed to requeue: #{e.message}" }
       end
 
-      def self.requeue_wip_jobs(queues, index)
+      def requeue_wip_jobs
         jobs_to_requeue = {}
         Sidekiq.redis do |conn|
-          queues.map { |q| "priority-queue:#{q}" }.each do |q|
-            wip_queue = "#{q}_#{Socket.gethostname}_#{index}"
+          @queues.each do |q|
+            wip_queue_name = wip_queue(q)
             jobs_to_requeue[q] = []
-            while job = conn.spop(wip_queue) do
+
+            while job = conn.spop(wip_queue_name)
               jobs_to_requeue[q] << job
             end
           end
 
           conn.pipelined do
             jobs_to_requeue.each do |queue, jobs|
-              return unless jobs.size > 0
-              conn.zadd(queue, jobs.map{|j| [0,j] })
+              next if jobs.empty? # ZADD doesn't work with empty arrays
+
+              conn.zadd(queue, jobs.map { |j| [0, j] })
             end
           end
         end
-        Sidekiq.logger.info("Pushed #{ jobs_to_requeue.map{|q| q.size }.reduce(:+) } jobs back to Redis")
-      rescue => ex
-        Sidekiq.logger.warn("Failed to requeue #{ jobs_to_requeue.map{|q| q.size }.reduce(:+) } jobs: #{ex.message}")
+        Sidekiq.logger.info("Priority ReliableFetch: Pushed #{jobs_to_requeue.values.flatten.size} jobs back to Redis")
+      rescue StandardError => e
+        Sidekiq.logger.warn("Priority ReliableFetch: Failed to requeue #{jobs_to_requeue.values.flatten.size} jobs: #{e.message}")
+      end
+
+      def register_myself
+        super_process_wip_queues = @queues.map { |q| wip_queue(q) }
+        id = identity # This is from standard sidekiq, updated with every heartbeat
+
+        # This method will run multiple times so seeing this message twice is no problem.
+        Sidekiq.logger.debug { "Priority ReliableFetch: Registering super process #{id} with #{super_process_wip_queues}" }
+
+        Sidekiq.redis do |conn|
+          conn.multi do
+            conn.sadd(SUPER_PROCESSES_REGISTRY_KEY, id)
+            conn.sadd("#{id}:super_priority_queues", super_process_wip_queues)
+          end
+        end
+      end
+
+      def unregister_super_process
+        id = identity
+        Sidekiq.logger.debug { "Priority ReliableFetch: Unregistering super process #{id}" }
+        Sidekiq.redis do |conn|
+          conn.multi do
+            conn.srem(SUPER_PROCESSES_REGISTRY_KEY, id)
+            conn.del("#{id}:super_priority_queues")
+          end
+        end
       end
     end
   end

data/sidekiq-priority_queue.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name        = 'sidekiq-priority_queue'
-  s.version     = '1.0.2'
+  s.version     = '1.0.6'
   s.date        = '2018-07-31'
   s.summary     = "Priority Queuing for Sidekiq"
   s.description = "An extension for Sidekiq allowing jobs in a single queue to be executed by a priority score rather than FIFO"
@@ -11,6 +11,6 @@ Gem::Specification.new do |s|
   s.license     = 'MIT'
   s.required_ruby_version = '>= 2.5.0'
 
-  s.add_dependency 'sidekiq', '>= 6'
+  s.add_dependency 'sidekiq', '>= 6.2.2'
   s.add_development_dependency 'minitest', '~> 5.10', '>= 5.10.1'
 end

metadata

@@ -1,12 +1,12 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-priority_queue
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.0.6
 platform: ruby
 authors:
 - Jacob Matthews
 - Petr Kopac
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
 date: 2018-07-31 00:00:00.000000000 Z
@@ -17,14 +17,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '6'
+        version: 6.2.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '6'
+        version: 6.2.2
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -78,7 +78,7 @@ homepage: https://github.com/chartmogul/sidekiq-priority_queue
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -93,8 +93,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
-signing_key: 
+rubygems_version: 3.2.3
+signing_key:
 specification_version: 4
 summary: Priority Queuing for Sidekiq
 test_files: []