sidekiq-priority_queue 1.0.3 → 1.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1d7a0cfd84d18380c50133959601b743de11149791f887badcc8436f7d015a88
-  data.tar.gz: 135fd23d2e9da9b325767059fed2fa227b891ade3d5de9d6fce95fca26fb75f4
+  metadata.gz: 041d042d2b08144431bc0e2ea76043e0f5f7e5cec5c28fc1c5a76b17c5390069
+  data.tar.gz: e2e2dc2e38375942d02a93740191a4aca744caa8d3462667dbd63780e0585ff6
 SHA512:
-  metadata.gz: 9535de17f1caa416733929806815348399e5871c51f1dbe114d164486ed9889379fbba7074ab31bce0e5c9c6d7c76e5824789b8c703c37a550a6c9f4e04660c9
-  data.tar.gz: a9d5d8f8a1688c6ef79d82fc30c9187f2cbd16658506b9be06a3750a62b2d6fdc5fe35f25eebb7535658f2a83fc369d22734cd8367a26a90eda98ee9857bb8aa
+  metadata.gz: b02c99baec57233cc4d315662faeb078d75d4bf887d43ec22cbec313bb4e1b7f2716236c1cbae27284871bd335d9701d26bb7ab9737438212e5448aa0ef2858a
+  data.tar.gz: 48e42a6e9eb50b28ae3446a4aacbdb3294fc734b48421fb4cbe86fc3add06b0feaf95c1eed3f60f1ae2a0e0e1b372c979f05097a8dbfbefd3c416143a0716c0c

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    sidekiq-priority_queue (1.0.3)
+    sidekiq-priority_queue (1.0.4)
       sidekiq (>= 6.2.2)
 
 GEM
@@ -23,7 +23,7 @@ GEM
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
     rake (13.0.3)
-    redis (4.4.0)
+    redis (4.5.1)
     sidekiq (6.2.2)
       connection_pool (>= 2.2.2)
       rack (~> 2.0)
@@ -47,4 +47,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.1.4
+   2.2.3

data/lib/sidekiq/priority_queue/reliable_fetch.rb

@@ -1,9 +1,11 @@
 # frozen_string_literal: true
 require 'sidekiq'
+require 'sidekiq/util'
 
 module Sidekiq
   module PriorityQueue
     class ReliableFetch
+      include Sidekiq::Util
 
       UnitOfWork = Struct.new(:queue, :job, :wip_queue) do
         def acknowledge
@@ -38,10 +40,26 @@ module Sidekiq
         @strictly_ordered_queues = !!options[:strict]
         @queues = options[:queues].map { |q| "priority-queue:#{q}" }
         @queues = @queues.uniq if @strictly_ordered_queues
+        @done = false
         @process_index = options[:index] || ENV['PROCESS_INDEX']
       end
 
+      def setup
+        Sidekiq.on(:startup) do
+          cleanup_the_dead
+          register_myself
+        end
+        Sidekiq.on(:shutdown) do
+          @done = true
+        end
+        Sidekiq.on(:heartbeat) do
+          register_myself
+        end
+      end
+
       def retrieve_work
+        return nil if @done
+
         work = @queues.detect do |q|
           job = zpopmin_sadd(q, wip_queue(q));
           break [q,job] if job
@@ -50,7 +68,7 @@ module Sidekiq
       end
 
       def wip_queue(q)
-        "#{q}_#{Socket.gethostname}_#{@process_index}"
+        "queue:spriorityq|#{identity}|#{q}"
       end
 
       def zpopmin_sadd(queue, wip_queue)
@@ -61,7 +79,7 @@ module Sidekiq
       end
 
       def spop(wip_queue)
-        Sidekiq.redis{ |con| con.spop(wip_queue) }
+        Sidekiq.redis { |con| con.spop(wip_queue) }
       end
 
       def queues_cmd
@@ -72,55 +90,107 @@ module Sidekiq
         end
       end
 
-      def bulk_requeue(_inprogress, options)
-        Sidekiq.logger.debug { "Re-queueing terminated jobs" }
-        process_index = options[:index] || ENV['PROCESS_INDEX']
-        self.class.requeue_wip_jobs(options[:queues], process_index)
+      # Below method is called when we close sidekiq process gracefully
+      def bulk_requeue(_inprogress, _options)
+        Sidekiq.logger.debug { "Priority ReliableFetch: Re-queueing terminated jobs" }
+        requeue_wip_jobs
+        unregister_super_process
       end
 
-      def self.resume_wip_jobs(queues, process_index)
-        Sidekiq.logger.debug { "Re-queueing WIP jobs" }
-        process_index ||= ENV['PROCESS_INDEX']
-        requeue_wip_jobs(queues, process_index)
-      end
+      private
 
-      Sidekiq.configure_server do |config|
-        config.on(:startup) do
-          if reliable_fetch_active?(config)
-            Sidekiq::PriorityQueue::ReliableFetch.resume_wip_jobs(config.options[:queues], config.options[:index])
+      # Below method is only to make sure we get jobs from incorrectly closed process (for example force killed using kill -9 SIDEKIQ_PID)
+      def cleanup_the_dead
+        overall_moved_count = 0
+        Sidekiq.redis do |conn|
+          conn.sscan_each("super_processes_priority") do |super_process|
+            next if conn.exists?(super_process) # Don't clean up currently running processes
+
+            Sidekiq.logger.debug { "Priority ReliableFetch: Moving job from #{super_process} back to original queues" }
+
+            # We need to pushback any leftover jobs still in WIP
+            previously_handled_queues = conn.smembers("#{super_process}:super_priority_queues")
+
+            # Below previously_handled_queues are simply WIP queues of previous, dead processes
+            previously_handled_queues.each do |previously_handled_queue|
+              queue_moved_size = 0
+              original_priority_queue_name = previously_handled_queue.split('|').last
+
+              Sidekiq.logger.debug { "Priority ReliableFetch: Moving job from #{previously_handled_queue} back to original queue: #{original_priority_queue_name}" }
+              loop do
+                break if conn.scard(previously_handled_queue) == 0
+
+                # Here we should wrap below two operations in Lua script
+                item = conn.spop(previously_handled_queue)
+                conn.zadd(original_priority_queue_name, 0, item)
+                queue_moved_size += 1
+                overall_moved_count += 1
+              end
+              # Below we simply remove old WIP queue
+              conn.srem(previously_handled_queue) if conn.scard(previously_handled_queue) == 0
+              Sidekiq.logger.debug { "Priority ReliableFetch: Moved #{queue_moved_size} jobs from ##{previously_handled_queue} back to original_queue: #{original_priority_queue_name} "}
+            end
+
+            Sidekiq.logger.debug { "Priority ReliableFetch: Unregistering super process #{super_process}" }
+            conn.del("#{super_process}:super_priority_queues")
+            conn.srem("super_processes_priority", super_process)
           end
         end
+        Sidekiq.logger.debug { "Priority ReliableFetch: Moved overall #{overall_moved_count} jobs from WIP queues" }
+      rescue => ex
+        # best effort, ignore Redis network errors
+        Sidekiq.logger.warn { "Priority ReliableFetch: Failed to requeue: #{ex.message}" }
       end
 
-      private
-
-      def self.reliable_fetch_active?(config)
-        return true if config.options[:fetch].is_a?(Sidekiq::PriorityQueue::ReliableFetch)
-        return config.options[:fetch].is_a?(Sidekiq::PriorityQueue::CombinedFetch) &&
-          config.options[:fetch].fetches.any? { |f| f.is_a?(Sidekiq::PriorityQueue::ReliableFetch) }
-      end
-
-      def self.requeue_wip_jobs(queues, index)
+      def requeue_wip_jobs
         jobs_to_requeue = {}
         Sidekiq.redis do |conn|
-          queues.map { |q| "priority-queue:#{q}" }.each do |q|
-            wip_queue = "#{q}_#{Socket.gethostname}_#{index}"
+          @queues.each do |q|
+            wip_queue_name = wip_queue(q)
             jobs_to_requeue[q] = []
-            while job = conn.spop(wip_queue) do
+
+            while job = conn.spop(wip_queue_name) do
               jobs_to_requeue[q] << job
             end
           end
 
           conn.pipelined do
             jobs_to_requeue.each do |queue, jobs|
-              return unless jobs.size > 0
-              conn.zadd(queue, jobs.map{|j| [0,j] })
+              next if jobs.size == 0 # ZADD doesn't work with empty arrays
+
+              conn.zadd(queue, jobs.map {|j| [0, j] })
             end
           end
         end
-        Sidekiq.logger.info("Pushed #{ jobs_to_requeue.map{|q| q.size }.reduce(:+) } jobs back to Redis")
+        Sidekiq.logger.info("Priority ReliableFetch: Pushed #{ jobs_to_requeue.values.flatten.size } jobs back to Redis")
       rescue => ex
-        Sidekiq.logger.warn("Failed to requeue #{ jobs_to_requeue.map{|q| q.size }.reduce(:+) } jobs: #{ex.message}")
+        Sidekiq.logger.warn("Priority ReliableFetch: Failed to requeue #{ jobs_to_requeue.values.flatten.size } jobs: #{ex.message}")
+      end
+
+      def register_myself
+        super_process_wip_queues = @queues.map { |q| wip_queue(q) }
+        id = identity # This is from standard sidekiq, updated with every heartbeat
+
+        # This method will run multiple times so seeing this message twice is no problem.
+        Sidekiq.logger.debug { "Priority ReliableFetch: Registering super process #{id} with #{super_process_wip_queues}" }
+
+        Sidekiq.redis do |conn|
+          conn.multi do
+            conn.sadd("super_processes_priority", id)
+            conn.sadd("#{id}:super_priority_queues", super_process_wip_queues)
+          end
+        end
+      end
+
+      def unregister_super_process
+        id = identity
+        Sidekiq.logger.debug { "Priority ReliableFetch: Unregistering super process #{id}" }
+        Sidekiq.redis do |conn|
+          conn.multi do
+            conn.srem("super_processes_priority", id)
+            conn.del("#{id}:super_priority_queues")
+          end
+        end
       end
     end
   end

data/sidekiq-priority_queue.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name        = 'sidekiq-priority_queue'
-  s.version     = '1.0.3'
+  s.version     = '1.0.4'
   s.date        = '2018-07-31'
   s.summary     = "Priority Queuing for Sidekiq"
   s.description = "An extension for Sidekiq allowing jobs in a single queue to be executed by a priority score rather than FIFO"

metadata

@@ -1,12 +1,12 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-priority_queue
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.0.4
 platform: ruby
 authors:
 - Jacob Matthews
 - Petr Kopac
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
 date: 2018-07-31 00:00:00.000000000 Z
@@ -78,7 +78,7 @@ homepage: https://github.com/chartmogul/sidekiq-priority_queue
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -93,8 +93,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
-signing_key: 
+rubygems_version: 3.2.3
+signing_key:
 specification_version: 4
 summary: Priority Queuing for Sidekiq
 test_files: []