sidekiq-priority_queue 1.0.5 → 1.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e6f96814c578dd2c2009fc89c069aa41df566f1fb81bc2b058b9d2fc8247fc9c
-  data.tar.gz: f55d5611495b8d61969508ab89124e0027685b182cd5ebdb8c72595417e534df
+  metadata.gz: 0e649f9936ca3989853559d9254ce6072229a700e5b5d4f17efc1dbf723c0cbe
+  data.tar.gz: d53da6399ee5f1405598de900b753ebbe3cb3b7b45e7ca2991b915c76ac1a3a8
 SHA512:
-  metadata.gz: '084d4acf2bff8504fadeefb141ca6617cbfa7ed8241cc1fbb93367d2ab844c1acaec11b771b89b933165b391400929b21176ffcc053ca1f5a4bc09807936f6ac'
-  data.tar.gz: c6628032befb4f253463df37ea0335d76434938b230891c51a779124e9f7bf5828e5e30b9de2c19c2a9ed76a02bf567059f737d797f335f3ca7b591e331354d6
+  metadata.gz: 9cf8bc3aca671561f792ce90592296655dba58541e0506c362c73ed3c54b42a8978235423c327f714b6f4749f9585ad29a942547ac5378621cfa1441bb215ef2
+  data.tar.gz: 8be1d8ad3d836104c19bc3e2473b6d77459ea3938557bd4672e72b92b3dc10a0e1cb2c2d94bc576e3c346c49a27528f94024bd9613603ef2e22de5ff1343d4ba

data/lib/sidekiq/priority_queue/reliable_fetch.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'sidekiq'
 require 'sidekiq/util'
 
@@ -7,6 +8,8 @@ module Sidekiq
     class ReliableFetch
       include Sidekiq::Util
 
+      SUPER_PROCESSES_REGISTRY_KEY = 'super_processes_priority'
+
       UnitOfWork = Struct.new(:queue, :job, :wip_queue) do
         def acknowledge
           Sidekiq.redis do |conn|
@@ -37,6 +40,7 @@ module Sidekiq
       end
 
       def initialize(options)
+        @options = options
         @strictly_ordered_queues = !!options[:strict]
         @queues = options[:queues].map { |q| "priority-queue:#{q}" }
         @queues = @queues.uniq if @strictly_ordered_queues
@@ -48,6 +52,7 @@ module Sidekiq
         Sidekiq.on(:startup) do
           cleanup_the_dead
           register_myself
+          check
         end
         Sidekiq.on(:shutdown) do
           @done = true
@@ -61,8 +66,8 @@ module Sidekiq
         return nil if @done
 
         work = @queues.detect do |q|
-          job = zpopmin_sadd(q, wip_queue(q));
-          break [q,job] if job
+          job = zpopmin_sadd(q, wip_queue(q))
+          break [q, job] if job
         end
         UnitOfWork.new(*work, wip_queue(work.first)) if work
       end
@@ -72,9 +77,9 @@ module Sidekiq
       end
 
       def zpopmin_sadd(queue, wip_queue)
-        Sidekiq.redis do |con|
-          @script_sha ||= con.script(:load, Sidekiq::PriorityQueue::Scripts::ZPOPMIN_SADD)
-          con.evalsha(@script_sha, [queue, wip_queue])
+        Sidekiq.redis do |conn|
+          @script_sha ||= conn.script(:load, Sidekiq::PriorityQueue::Scripts::ZPOPMIN_SADD)
+          conn.evalsha(@script_sha, [queue, wip_queue])
         end
       end
 
@@ -92,18 +97,80 @@ module Sidekiq
 
       # Below method is called when we close sidekiq process gracefully
       def bulk_requeue(_inprogress, _options)
-        Sidekiq.logger.debug { "Priority ReliableFetch: Re-queueing terminated jobs" }
+        Sidekiq.logger.debug { 'Priority ReliableFetch: Re-queueing terminated jobs' }
         requeue_wip_jobs
         unregister_super_process
       end
 
       private
 
+      def check
+        check_for_orphans if orphan_check?
+      rescue StandardError => e
+        # orphan check is best effort, we don't want Redis downtime to
+        # break Sidekiq
+        Sidekiq.logger.warn { "Priority ReliableFetch: Failed to do orphan check: #{e.message}" }
+      end
+
+      def orphan_check?
+        delay = @options.fetch(:reliable_fetch_orphan_check, 3600).to_i
+        return false if delay.zero?
+
+        Sidekiq.redis do |conn|
+          conn.set('priority_reliable_fetch_orphan_check', Time.now.to_f, ex: delay, nx: true)
+        end
+      end
+
+      # This method is extra paranoid verification to check Redis for any possible
+      # orphaned queues with jobs. If we change queue names and lose jobs in the meantime,
+      # this will find old queues with jobs and rescue them.
+      def check_for_orphans
+        orphans_count = 0
+        queues_count = 0
+        orphan_queues = Set.new
+        Sidekiq.redis do |conn|
+          ids = conn.smembers(SUPER_PROCESSES_REGISTRY_KEY)
+          Sidekiq.logger.debug("Priority ReliableFetch found #{ids.size} super processes")
+
+          conn.scan_each(match: 'queue:spriorityq|*', count: 100) do |wip_queue|
+            queues_count += 1
+            _, id, original_priority_queue_name = wip_queue.split('|')
+            next if ids.include?(id)
+
+            # Race condition in pulling super_processes and checking queue liveness.
+            # Need to verify in Redis.
+            unless conn.sismember(SUPER_PROCESSES_REGISTRY_KEY, id)
+              orphan_queues << original_priority_queue_name
+              queue_jobs_count = 0
+              loop do
+                break if conn.scard(wip_queue).zero?
+
+                # Here we should wrap below two operations in Lua script
+                item = conn.spop(wip_queue)
+                conn.zadd(original_priority_queue_name, 0, item)
+                orphans_count += 1
+                queue_jobs_count += 1
+              end
+              if queue_jobs_count.positive?
+                Sidekiq::Pro.metrics.increment('jobs.recovered.fetch', by: queue_jobs_count, tags: ["queue:#{original_priority_queue_name}"])
+              end
+            end
+          end
+        end
+
+        if orphans_count.positive?
+          Sidekiq.logger.warn { "Priority ReliableFetch recovered #{orphans_count} orphaned jobs in queues: #{orphan_queues.to_a.inspect}" }
+        elsif queues_count.positive?
+          Sidekiq.logger.info { "Priority ReliableFetch found #{queues_count} working queues with no orphaned jobs" }
+        end
+        orphans_count
+      end
+
       # Below method is only to make sure we get jobs from incorrectly closed process (for example force killed using kill -9 SIDEKIQ_PID)
       def cleanup_the_dead
         overall_moved_count = 0
         Sidekiq.redis do |conn|
-          conn.sscan_each("super_processes_priority") do |super_process|
+          conn.sscan_each(SUPER_PROCESSES_REGISTRY_KEY) do |super_process|
             next if conn.exists?(super_process) # Don't clean up currently running processes
 
             Sidekiq.logger.debug { "Priority ReliableFetch: Moving job from #{super_process} back to original queues" }
@@ -118,7 +185,7 @@ module Sidekiq
 
               Sidekiq.logger.debug { "Priority ReliableFetch: Moving job from #{previously_handled_queue} back to original queue: #{original_priority_queue_name}" }
               loop do
-                break if conn.scard(previously_handled_queue) == 0
+                break if conn.scard(previously_handled_queue).zero?
 
                 # Here we should wrap below two operations in Lua script
                 item = conn.spop(previously_handled_queue)
@@ -127,19 +194,19 @@ module Sidekiq
                 overall_moved_count += 1
               end
               # Below we simply remove old WIP queue
-              conn.del(previously_handled_queue) if conn.scard(previously_handled_queue) == 0
-              Sidekiq.logger.debug { "Priority ReliableFetch: Moved #{queue_moved_size} jobs from ##{previously_handled_queue} back to original_queue: #{original_priority_queue_name} "}
+              conn.del(previously_handled_queue) if conn.scard(previously_handled_queue).zero?
+              Sidekiq.logger.debug { "Priority ReliableFetch: Moved #{queue_moved_size} jobs from ##{previously_handled_queue} back to original_queue: #{original_priority_queue_name} " }
             end
 
             Sidekiq.logger.debug { "Priority ReliableFetch: Unregistering super process #{super_process}" }
             conn.del("#{super_process}:super_priority_queues")
-            conn.srem("super_processes_priority", super_process)
+            conn.srem(SUPER_PROCESSES_REGISTRY_KEY, super_process)
           end
         end
         Sidekiq.logger.debug { "Priority ReliableFetch: Moved overall #{overall_moved_count} jobs from WIP queues" }
-      rescue => ex
+      rescue StandardError => e
         # best effort, ignore Redis network errors
-        Sidekiq.logger.warn { "Priority ReliableFetch: Failed to requeue: #{ex.message}" }
+        Sidekiq.logger.warn { "Priority ReliableFetch: Failed to requeue: #{e.message}" }
       end
 
       def requeue_wip_jobs
@@ -149,22 +216,22 @@ module Sidekiq
             wip_queue_name = wip_queue(q)
             jobs_to_requeue[q] = []
 
-            while job = conn.spop(wip_queue_name) do
+            while job = conn.spop(wip_queue_name)
               jobs_to_requeue[q] << job
             end
           end
 
           conn.pipelined do
             jobs_to_requeue.each do |queue, jobs|
-              next if jobs.size == 0 # ZADD doesn't work with empty arrays
+              next if jobs.empty? # ZADD doesn't work with empty arrays
 
-              conn.zadd(queue, jobs.map {|j| [0, j] })
+              conn.zadd(queue, jobs.map { |j| [0, j] })
             end
           end
         end
-        Sidekiq.logger.info("Priority ReliableFetch: Pushed #{ jobs_to_requeue.values.flatten.size } jobs back to Redis")
-      rescue => ex
-        Sidekiq.logger.warn("Priority ReliableFetch: Failed to requeue #{ jobs_to_requeue.values.flatten.size } jobs: #{ex.message}")
+        Sidekiq.logger.info("Priority ReliableFetch: Pushed #{jobs_to_requeue.values.flatten.size} jobs back to Redis")
+      rescue StandardError => e
+        Sidekiq.logger.warn("Priority ReliableFetch: Failed to requeue #{jobs_to_requeue.values.flatten.size} jobs: #{e.message}")
       end
 
       def register_myself
@@ -176,7 +243,7 @@ module Sidekiq
 
         Sidekiq.redis do |conn|
           conn.multi do
-            conn.sadd("super_processes_priority", id)
+            conn.sadd(SUPER_PROCESSES_REGISTRY_KEY, id)
             conn.sadd("#{id}:super_priority_queues", super_process_wip_queues)
           end
         end
@@ -187,7 +254,7 @@ module Sidekiq
         Sidekiq.logger.debug { "Priority ReliableFetch: Unregistering super process #{id}" }
         Sidekiq.redis do |conn|
           conn.multi do
-            conn.srem("super_processes_priority", id)
+            conn.srem(SUPER_PROCESSES_REGISTRY_KEY, id)
             conn.del("#{id}:super_priority_queues")
           end
         end

data/sidekiq-priority_queue.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name        = 'sidekiq-priority_queue'
-  s.version     = '1.0.5'
+  s.version     = '1.0.6'
   s.date        = '2018-07-31'
   s.summary     = "Priority Queuing for Sidekiq"
   s.description = "An extension for Sidekiq allowing jobs in a single queue to be executed by a priority score rather than FIFO"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-priority_queue
 version: !ruby/object:Gem::Version
-  version: 1.0.5
+  version: 1.0.6
 platform: ruby
 authors:
 - Jacob Matthews