sidekiq-limit_fetch 4.3.2 → 4.4.1

data/lib/sidekiq/limit_fetch/global/monitor.rb

@@ -1,72 +1,83 @@
-module Sidekiq::LimitFetch::Global
-  module Monitor
-    extend self
+# frozen_string_literal: true
 
-    HEARTBEAT_PREFIX = 'limit:heartbeat:'
-    PROCESS_SET = 'limit:processes'
-    HEARTBEAT_TTL = 20
-    REFRESH_TIMEOUT = 5
+module Sidekiq
+  module LimitFetch
+    module Global
+      module Monitor
+        extend self
 
-    def start!(ttl=HEARTBEAT_TTL, timeout=REFRESH_TIMEOUT)
-      Thread.new do
-        loop do
-          Sidekiq::LimitFetch.redis_retryable do
-            handle_dynamic_queues
-            update_heartbeat ttl
-            invalidate_old_processes
+        HEARTBEAT_PREFIX = 'limit:heartbeat:'
+        PROCESS_SET = 'limit:processes'
+        HEARTBEAT_TTL = 20
+        REFRESH_TIMEOUT = 5
+
+        def start!(ttl = HEARTBEAT_TTL, timeout = REFRESH_TIMEOUT)
+          Thread.new do
+            loop do
+              Sidekiq::LimitFetch.redis_retryable do
+                handle_dynamic_queues
+                update_heartbeat ttl
+                invalidate_old_processes
+              end
+
+              sleep timeout
+            end
           end
+        end
 
-          sleep timeout
+        def all_processes
+          Sidekiq.redis { |it| it.smembers PROCESS_SET }
         end
-      end
-    end
 
-    def all_processes
-      Sidekiq.redis {|it| it.smembers PROCESS_SET }
-    end
+        def old_processes
+          all_processes.reject do |process|
+            Sidekiq.redis { |it| it.get heartbeat_key process } == '1'
+          end
+        end
 
-    def old_processes
-      all_processes.reject do |process|
-        Sidekiq.redis {|it| it.get heartbeat_key process }
-      end
-    end
+        def remove_old_processes!
+          Sidekiq.redis do |it|
+            old_processes.each { |process| it.srem PROCESS_SET, [process] }
+          end
+        end
 
-    def remove_old_processes!
-      Sidekiq.redis do |it|
-        old_processes.each {|process| it.srem PROCESS_SET, [process] }
-      end
-    end
+        def handle_dynamic_queues
+          queues = Sidekiq::LimitFetch::Queues
+          return unless queues.dynamic?
 
-    def handle_dynamic_queues
-      queues = Sidekiq::LimitFetch::Queues
-      queues.handle Sidekiq::Queue.all.map(&:name) if queues.dynamic?
-    end
+          available_queues = Sidekiq::Queue.all.map(&:name).reject do |it|
+            queues.dynamic_exclude.include? it
+          end
+          queues.handle available_queues
+        end
 
-    private
+        private
 
-    def update_heartbeat(ttl)
-      Sidekiq.redis do |it|
-        it.multi do |pipeline|
-          pipeline.set heartbeat_key, true
-          pipeline.sadd PROCESS_SET, [Selector.uuid]
-          pipeline.expire heartbeat_key, ttl
+        def update_heartbeat(ttl)
+          Sidekiq.redis do |it|
+            it.multi do |pipeline|
+              pipeline.set heartbeat_key, '1'
+              pipeline.sadd PROCESS_SET, [Selector.uuid]
+              pipeline.expire heartbeat_key, ttl
+            end
+          end
         end
-      end
-    end
 
-    def invalidate_old_processes
-      Sidekiq.redis do |it|
-        remove_old_processes!
-        processes = all_processes
+        def invalidate_old_processes
+          Sidekiq.redis do |_it|
+            remove_old_processes!
+            processes = all_processes
 
-        Sidekiq::Queue.instances.each do |queue|
-          queue.remove_locks_except! processes
+            Sidekiq::Queue.instances.each do |queue|
+              queue.remove_locks_except! processes
+            end
+          end
         end
-      end
-    end
 
-    def heartbeat_key(process=Selector.uuid)
-      HEARTBEAT_PREFIX + process
+        def heartbeat_key(process = Selector.uuid)
+          HEARTBEAT_PREFIX + process
+        end
+      end
     end
   end
 end

data/lib/sidekiq/limit_fetch/global/selector.rb

@@ -1,48 +1,55 @@
-module Sidekiq::LimitFetch::Global
-  module Selector
-    extend self
+# frozen_string_literal: true
 
-    MUTEX_FOR_UUID = Mutex.new
+module Sidekiq
+  module LimitFetch
+    module Global
+      module Selector
+        extend self
 
-    def acquire(queues, namespace)
-      redis_eval :acquire, [namespace, uuid, queues]
-    end
+        MUTEX_FOR_UUID = Mutex.new
 
-    def release(queues, namespace)
-      redis_eval :release, [namespace, uuid, queues]
-    end
+        def acquire(queues, namespace)
+          redis_eval :acquire, [namespace, uuid, queues]
+        end
 
-    def uuid
-      # - if we'll remove "@uuid ||=" from inside of mutex
-      # then @uuid can be overwritten
-      # - if we'll remove "@uuid ||=" from outside of mutex
-      # then each read will lead to mutex
-      @uuid ||= MUTEX_FOR_UUID.synchronize { @uuid || SecureRandom.uuid }
-    end
+        def release(queues, namespace)
+          redis_eval :release, [namespace, uuid, queues]
+        end
 
-    private
+        def uuid
+          # - if we'll remove "@uuid ||=" from inside of mutex
+          # then @uuid can be overwritten
+          # - if we'll remove "@uuid ||=" from outside of mutex
+          # then each read will lead to mutex
+          @uuid ||= MUTEX_FOR_UUID.synchronize { @uuid || SecureRandom.uuid }
+        end
+
+        private
+
+        def redis_eval(script_name, args)
+          Sidekiq.redis do |it|
+            it.evalsha send("redis_#{script_name}_sha"), [], args
+          rescue Sidekiq::LimitFetch::RedisCommandError => e
+            raise unless e.message.include? 'NOSCRIPT'
 
-    def redis_eval(script_name, args)
-      Sidekiq.redis do |it|
-        begin
-          it.evalsha send("redis_#{script_name}_sha"), argv: args
-        rescue Redis::CommandError => error
-          raise unless error.message.include? 'NOSCRIPT'
-          it.eval send("redis_#{script_name}_script"), argv: args
+            if Sidekiq::LimitFetch.post_7?
+              it.eval send("redis_#{script_name}_script"), 0, *args
+            else
+              it.eval send("redis_#{script_name}_script"), argv: args
+            end
+          end
         end
-      end
-    end
 
-    def redis_acquire_sha
-      @acquire_sha ||= Digest::SHA1.hexdigest redis_acquire_script
-    end
+        def redis_acquire_sha
+          @redis_acquire_sha ||= OpenSSL::Digest::SHA1.hexdigest redis_acquire_script
+        end
 
-    def redis_release_sha
-      @release_sha ||= Digest::SHA1.hexdigest redis_release_script
-    end
+        def redis_release_sha
+          @redis_release_sha ||= OpenSSL::Digest::SHA1.hexdigest redis_release_script
+        end
 
-    def redis_acquire_script
-      <<-LUA
+        def redis_acquire_script
+          <<-LUA
         local namespace   = table.remove(ARGV, 1)..'limit_fetch:'
         local worker_name = table.remove(ARGV, 1)
         local queues      = ARGV
@@ -106,11 +113,11 @@ module Sidekiq::LimitFetch::Global
         end
 
         return available
-      LUA
-    end
+          LUA
+        end
 
-    def redis_release_script
-      <<-LUA
+        def redis_release_script
+          <<-LUA
         local namespace   = table.remove(ARGV, 1)..'limit_fetch:'
         local worker_name = table.remove(ARGV, 1)
         local queues      = ARGV
@@ -119,7 +126,9 @@ module Sidekiq::LimitFetch::Global
           local probed_key = namespace..'probed:'..queue
           redis.call('lrem', probed_key, 1, worker_name)
         end
-      LUA
+          LUA
+        end
+      end
     end
   end
 end

data/lib/sidekiq/limit_fetch/global/semaphore.rb

@@ -1,136 +1,141 @@
-module Sidekiq::LimitFetch::Global
-  class Semaphore
-    PREFIX = 'limit_fetch'
+# frozen_string_literal: true
 
-    attr_reader :local_busy
+module Sidekiq
+  module LimitFetch
+    module Global
+      class Semaphore
+        PREFIX = 'limit_fetch'
 
-    def initialize(name)
-      @name = name
-      @lock = Mutex.new
-      @local_busy = 0
-    end
+        attr_reader :local_busy
 
-    def limit
-      value = redis {|it| it.get "#{PREFIX}:limit:#@name" }
-      value.to_i if value
-    end
+        def initialize(name)
+          @name = name
+          @lock = Mutex.new
+          @local_busy = 0
+        end
 
-    def limit=(value)
-      @limit_changed = true
+        def limit
+          value = redis { |it| it.get "#{PREFIX}:limit:#{@name}" }
+          value&.to_i
+        end
 
-      if value
-        redis {|it| it.set "#{PREFIX}:limit:#@name", value }
-      else
-        redis {|it| it.del "#{PREFIX}:limit:#@name" }
-      end
-    end
+        def limit=(value)
+          @limit_changed = true
 
-    def limit_changed?
-      @limit_changed
-    end
+          if value
+            redis { |it| it.set "#{PREFIX}:limit:#{@name}", value }
+          else
+            redis { |it| it.del "#{PREFIX}:limit:#{@name}" }
+          end
+        end
 
-    def process_limit
-      value = redis {|it| it.get "#{PREFIX}:process_limit:#@name" }
-      value.to_i if value
-    end
+        def limit_changed?
+          @limit_changed
+        end
 
-    def process_limit=(value)
-      if value
-        redis {|it| it.set "#{PREFIX}:process_limit:#@name", value }
-      else
-        redis {|it| it.del "#{PREFIX}:process_limit:#@name" }
-      end
-    end
+        def process_limit
+          value = redis { |it| it.get "#{PREFIX}:process_limit:#{@name}" }
+          value&.to_i
+        end
 
-    def acquire
-      Selector.acquire([@name], namespace).size > 0
-    end
+        def process_limit=(value)
+          if value
+            redis { |it| it.set "#{PREFIX}:process_limit:#{@name}", value }
+          else
+            redis { |it| it.del "#{PREFIX}:process_limit:#{@name}" }
+          end
+        end
 
-    def release
-      redis {|it| it.lrem "#{PREFIX}:probed:#@name", 1, Selector.uuid }
-    end
+        def acquire
+          Selector.acquire([@name], namespace).size.positive?
+        end
 
-    def busy
-      redis {|it| it.llen "#{PREFIX}:busy:#@name" }
-    end
+        def release
+          redis { |it| it.lrem "#{PREFIX}:probed:#{@name}", 1, Selector.uuid }
+        end
 
-    def busy_processes
-      redis {|it| it.lrange "#{PREFIX}:busy:#@name", 0, -1 }
-    end
+        def busy
+          redis { |it| it.llen "#{PREFIX}:busy:#{@name}" }
+        end
 
-    def increase_busy
-      increase_local_busy
-      redis {|it| it.rpush "#{PREFIX}:busy:#@name", Selector.uuid }
-    end
+        def busy_processes
+          redis { |it| it.lrange "#{PREFIX}:busy:#{@name}", 0, -1 }
+        end
 
-    def decrease_busy
-      decrease_local_busy
-      redis {|it| it.lrem "#{PREFIX}:busy:#@name", 1, Selector.uuid }
-    end
+        def increase_busy
+          increase_local_busy
+          redis { |it| it.rpush "#{PREFIX}:busy:#{@name}", Selector.uuid }
+        end
 
-    def probed
-      redis {|it| it.llen "#{PREFIX}:probed:#@name" }
-    end
+        def decrease_busy
+          decrease_local_busy
+          redis { |it| it.lrem "#{PREFIX}:busy:#{@name}", 1, Selector.uuid }
+        end
 
-    def probed_processes
-      redis {|it| it.lrange "#{PREFIX}:probed:#@name", 0, -1 }
-    end
+        def probed
+          redis { |it| it.llen "#{PREFIX}:probed:#{@name}" }
+        end
 
-    def pause
-      redis {|it| it.set "#{PREFIX}:pause:#@name", true }
-    end
+        def probed_processes
+          redis { |it| it.lrange "#{PREFIX}:probed:#{@name}", 0, -1 }
+        end
 
-    def pause_for_ms ms
-      redis {|it| it.psetex "#{PREFIX}:pause:#@name", ms, true }
-    end
+        def pause
+          redis { |it| it.set "#{PREFIX}:pause:#{@name}", '1' }
+        end
 
-    def unpause
-      redis {|it| it.del "#{PREFIX}:pause:#@name" }
-    end
+        def pause_for_ms(milliseconds)
+          redis { |it| it.psetex "#{PREFIX}:pause:#{@name}", milliseconds, 1 }
+        end
 
-    def paused?
-      redis {|it| it.get "#{PREFIX}:pause:#@name" }
-    end
+        def unpause
+          redis { |it| it.del "#{PREFIX}:pause:#{@name}" }
+        end
 
-    def block
-      redis {|it| it.set "#{PREFIX}:block:#@name", true }
-    end
+        def paused?
+          redis { |it| it.get "#{PREFIX}:pause:#{@name}" } == '1'
+        end
 
-    def block_except(*queues)
-      raise ArgumentError if queues.empty?
-      redis {|it| it.set "#{PREFIX}:block:#@name", queues.join(',') }
-    end
+        def block
+          redis { |it| it.set "#{PREFIX}:block:#{@name}", '1' }
+        end
 
-    def unblock
-      redis {|it| it.del "#{PREFIX}:block:#@name" }
-    end
+        def block_except(*queues)
+          raise ArgumentError if queues.empty?
 
-    def blocking?
-      redis {|it| it.get "#{PREFIX}:block:#@name" }
-    end
+          redis { |it| it.set "#{PREFIX}:block:#{@name}", queues.join(',') }
+        end
 
-    def clear_limits
-      redis do |it|
-        %w(block busy limit pause probed process_limit).each do |key|
-          it.del "#{PREFIX}:#{key}:#@name"
+        def unblock
+          redis { |it| it.del "#{PREFIX}:block:#{@name}" }
         end
-      end
-    end
 
-    def increase_local_busy
-      @lock.synchronize { @local_busy += 1 }
-    end
+        def blocking?
+          redis { |it| it.get "#{PREFIX}:block:#{@name}" } == '1'
+        end
 
-    def decrease_local_busy
-      @lock.synchronize { @local_busy -= 1 }
-    end
+        def clear_limits
+          redis do |it|
+            %w[block busy limit pause probed process_limit].each do |key|
+              it.del "#{PREFIX}:#{key}:#{@name}"
+            end
+          end
+        end
 
-    def local_busy?
-      @local_busy > 0
-    end
+        def increase_local_busy
+          @lock.synchronize { @local_busy += 1 }
+        end
+
+        def decrease_local_busy
+          @lock.synchronize { @local_busy -= 1 }
+        end
+
+        def local_busy?
+          @local_busy.positive?
+        end
 
-    def explain
-      <<-END.gsub(/^ {8}/, '')
+        def explain
+          <<-INFO.gsub(/^ {8}/, '')
         Current sidekiq process: #{Selector.uuid}
 
           All processes:
@@ -153,31 +158,33 @@ module Sidekiq::LimitFetch::Global
 
           Blocking:
         #{blocking?}
-      END
-    end
+          INFO
+        end
 
-    def remove_locks_except!(processes)
-      locked_processes = probed_processes.uniq
-      (locked_processes - processes).each do |dead_process|
-        remove_lock! dead_process
-      end
-    end
+        def remove_locks_except!(processes)
+          locked_processes = probed_processes.uniq
+          (locked_processes - processes).each do |dead_process|
+            remove_lock! dead_process
+          end
+        end
 
-    def remove_lock!(process)
-      redis do |it|
-        it.lrem "#{PREFIX}:probed:#@name", 0, process
-        it.lrem "#{PREFIX}:busy:#@name", 0, process
-      end
-    end
+        def remove_lock!(process)
+          redis do |it|
+            it.lrem "#{PREFIX}:probed:#{@name}", 0, process
+            it.lrem "#{PREFIX}:busy:#{@name}", 0, process
+          end
+        end
 
-    private
+        private
 
-    def redis(&block)
-      Sidekiq.redis(&block)
-    end
+        def redis(&block)
+          Sidekiq.redis(&block)
+        end
 
-    def namespace
-      Sidekiq::LimitFetch::Queues.namespace
+        def namespace
+          Sidekiq::LimitFetch::Queues.namespace
+        end
+      end
     end
   end
 end

data/lib/sidekiq/limit_fetch/instances.rb

@@ -1,23 +1,29 @@
-module Sidekiq::LimitFetch::Instances
-  def self.extended(klass)
-    klass.instance_variable_set :@instances, {}
-  end
+# frozen_string_literal: true
 
-  def new(*args)
-    @instances[args] ||= super
-  end
+module Sidekiq
+  module LimitFetch
+    module Instances
+      def self.extended(klass)
+        klass.instance_variable_set :@instances, {}
+      end
 
-  alias [] new
+      def new(*args)
+        @instances[args] ||= super
+      end
 
-  def instances
-    @instances.values
-  end
+      alias [] new
 
-  def reset_instances!
-    @instances = {}
-  end
+      def instances
+        @instances.values
+      end
+
+      def reset_instances!
+        @instances = {}
+      end
 
-  def delete_instance(name)
-    @instances.delete [name]
+      def delete_instance(name)
+        @instances.delete [name]
+      end
+    end
   end
 end