sidekiq-max-jobs 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 88185f0bd0aa5075fae604f768a9b011edf6c4d349bb8d5fe32a177488bca55a
-  data.tar.gz: a0fde7488aa0115cd4402c07508922cfe91faa7cf57e5589297432522f3fd97f
+  metadata.gz: 9bf624b8b84bbfd78039b22826e440c7218c9850207a55a1a24bfc046a0812a4
+  data.tar.gz: 364ca8c57ac52151636f7c9a0feb7ffe18d758fd95772fb692b0802fb634bef6
 SHA512:
-  metadata.gz: 675e4bb41052bfd805c2abb42686cad52bcb6a3fd8f0e70dac7974571b495dc8a3acb7fd3a4f249badc845e8c83ba2423034345d683ad0f0da19ee021b983753
-  data.tar.gz: 36742d445b462d6539a11c4d99e35503d32754d70c0096b9825593d1dac0ddee5815b6adadd7d6e98a6d07b1afc167f9f74d7ba424fc7e4ebafe16eb87c98fb7
+  metadata.gz: 12548204b50a4f9d3dfec6470d3488b03fc672c7c41c12e6554ed350004ce63b42d322043c21dfc023fbd6fc819a724c5a167d62494a0615be0c8d4e71bfda19
+  data.tar.gz: 7e704fde12c9f64b200ac001f6824821ec6e0698235d4493e01a30ff49f27e3742373ae38f0ef91c013b5ee08986bf619cb6c2922eff44c79b3898444b4c3984

data/.rubocop.yml

@@ -12,6 +12,9 @@ AllCops:
 Layout/DotPosition:
   Enabled: false
 
+Layout/LineLength:
+  Max: 120
+
 Layout/SpaceInLambdaLiteral:
   EnforcedStyle: require_space
 
@@ -19,9 +22,6 @@ Metrics/BlockLength:
   Exclude:
     - 'spec/**/*'
 
-Metrics/LineLength:
-  Max: 120
-
 Style/Documentation:
   Enabled: false
 

data/.rubocop_todo.yml

@@ -1,6 +1,6 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2020-06-17 19:26:25 -0500 using RuboCop version 0.85.1.
+# on 2020-06-17 22:45:10 -0500 using RuboCop version 0.85.1.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
@@ -14,7 +14,7 @@ Lint/RescueException:
 # Offense count: 1
 # Configuration parameters: IgnoredMethods.
 Metrics/AbcSize:
-  Max: 19
+  Max: 27
 
 # Offense count: 1
 # Configuration parameters: CountComments, ExcludedMethods.
@@ -25,12 +25,17 @@ Metrics/BlockLength:
 # Offense count: 1
 # Configuration parameters: CountComments.
 Metrics/ClassLength:
-  Max: 102
+  Max: 152
+
+# Offense count: 1
+# Configuration parameters: IgnoredMethods.
+Metrics/CyclomaticComplexity:
+  Max: 7
 
 # Offense count: 1
 # Configuration parameters: CountComments, ExcludedMethods.
 Metrics/MethodLength:
-  Max: 17
+  Max: 24
 
 # Offense count: 1
 # Cop supports --auto-correct.

data/README.md

@@ -52,7 +52,7 @@ end
 If everything above is successful the next time you start your worker you will
 see a message like the following:
 ```bash
-2020-06-10T00:23:31.789Z pid=73703 tid=oxifk6l13 INFO: Max-Jobs middleware enabled, shutting down pid: 73703 after: 100 job(s)
+2020-06-10T00:23:31.789Z pid=73703 tid=oxifk6l13 INFO: Max-Jobs middleware enabled, shutting down pid: 73703 when max-jobs quota is reached
 ```
 
 Configuration Options
@@ -75,6 +75,18 @@ number between 1 and the value specified. This value is added to the
 of your `Worker(s)` restart at / around the same time (default:
 `rand((ENV['MAX_JOBS_JITTER'] || 1).to_i)`)
 
+Important Note
+--------------
+
+When determining if the max-job quota has been reached the total jobs processed
+is checked first, followed by the jobs processed for the current queue. If your
+`Worker(s)` are handling multiple queues it is generally recommended that you
+set the total value to the same value as your highest queue value (e.g. if you
+had `MAX_JOBS_FOO=100` and `MAX_JOBS_BAR=200` it probably makes sense to set
+`MAX_JOBS=200`, if not a little bit lower). Setting the right limits ultimately
+depends on the intensity / resource needs of the work being performed. The same
+rule of thumb applies to `MAX_JOBS_JITTER` as well.
+
 Contributing
 ------------
 

data/VERSION

@@ -1 +1 @@
-0.0.4
+0.0.5

data/lib/sidekiq/middleware/server/max_jobs.rb

@@ -20,22 +20,40 @@ module Sidekiq
             @cache ||= {}
           end
 
-          def counter(queue)
-            key = counter_key(queue)
-            return cache[key] if cache.include?(key)
+          def counter
+            key = counter_key
+            cache[key] ||= 0
+          end
 
-            cache[key] = 0
+          def counter_for_queue(queue)
+            key = counter_for_queue_key(queue)
+            cache[key] ||= 0
           end
 
-          def counter_key(queue)
+          def counter_for_queue_key(queue)
             "COUNTER_#{queue.upcase}"
           end
 
-          def increment_counter!(queue)
-            key = counter_key(queue)
-            counter = cache[key] || 0
+          def counter_key
+            'COUNTER'
+          end
+
+          def default_max_jobs
+            100
+          end
+
+          def default_max_jobs_jitter
+            1
+          end
+
+          def increment_counter!
+            key = counter_key
+            cache[key] = (cache[key] || 0).next
+          end
 
-            cache[key] = counter.next
+          def increment_counter_for_queue!(queue)
+            key = counter_for_queue_key(queue)
+            cache[key] = (cache[key] || 0).next
           end
 
           def log_info(message)
@@ -43,65 +61,98 @@ module Sidekiq
           end
 
           def log_initialization!
-            log_info("Max-Jobs middleware enabled, shutting down pid: #{pid} after max-jobs threshold reached")
+            log_info("Max-Jobs middleware enabled, shutting down pid: #{pid} when max-jobs quota is reached")
           end
 
-          def max_jobs(queue)
-            key = max_jobs_key(queue)
-            return cache[key] if cache.include?(key)
+          def max_jobs
+            key = max_jobs_key
+            cache[key] ||= (ENV[key] || default_max_jobs).to_i
+          end
 
-            cache[key] = (
+          def max_jobs_for_queue(queue)
+            key = max_jobs_for_queue_key(queue)
+            cache[key] ||= (
               ENV[key] ||
-              ENV['MAX_JOBS'] ||
-              100
+              ENV[max_jobs_key] ||
+              default_max_jobs
             ).to_i
           end
 
-          def max_jobs_key(queue)
+          def max_jobs_for_queue_key(queue)
             "MAX_JOBS_#{queue.upcase}"
           end
 
-          def max_jobs_jitter(queue)
-            key = max_jobs_jitter_key(queue)
-            return cache[key] if cache.include?(key)
+          def max_jobs_jitter
+            key = max_jobs_jitter_key
+            cache[key] ||= rand((ENV[key] || default_max_jobs_jitter).to_i)
+          end
 
-            cache[key] = rand(
+          def max_jobs_jitter_for_queue(queue)
+            key = max_jobs_jitter_for_queue_key(queue)
+            cache[key] ||= rand(
               (
                 ENV[key] ||
-                ENV['MAX_JOBS_JITTER'] ||
-                1
+                ENV[max_jobs_jitter_key] ||
+                default_max_jobs_jitter
               ).to_i
             )
           end
 
-          def max_jobs_jitter_key(queue)
+          def max_jobs_jitter_for_queue_key(queue)
             "MAX_JOBS_JITTER_#{queue.upcase}"
           end
 
-          def max_jobs_with_jitter(queue)
-            key = max_jobs_with_jitter_key(queue)
-            return cache[key] if cache.include?(key)
+          def max_jobs_jitter_key
+            'MAX_JOBS_JITTER'
+          end
 
-            cache[key] = max_jobs(queue) + max_jobs_jitter(queue)
+          def max_jobs_key
+            'MAX_JOBS'
           end
 
-          def max_jobs_with_jitter_key(queue)
+          def max_jobs_with_jitter
+            key = max_jobs_with_jitter_key
+            cache[key] ||= (max_jobs + max_jobs_jitter)
+          end
+
+          def max_jobs_with_jitter_for_queue(queue)
+            key = max_jobs_with_jitter_for_queue_key(queue)
+            cache[key] ||= \
+              (max_jobs_for_queue(queue) + max_jobs_jitter_for_queue(queue))
+          end
+
+          def max_jobs_with_jitter_for_queue_key(queue)
             "MAX_JOBS_WITH_JITTER_#{queue.upcase}"
           end
 
-          def mutex(queue)
-            key = mutex_key(queue)
-            return cache[key] if cache.include?(key)
+          def max_jobs_with_jitter_key
+            'MAX_JOBS_WITH_JITTER'
+          end
 
-            cache[key] = ::Mutex.new
+          def mutex
+            key = mutex_key
+            cache[key] ||= ::Mutex.new
           end
 
-          def mutex_key(queue)
-            "MUTEX_#{queue.upcase}"
+          def mutex_key
+            'MUTEX'
           end
 
           def pid
-            @pid ||= ::Process.pid
+            key = pid_key
+            cache[key] ||= ::Process.pid
+          end
+
+          def pid_key
+            'PID'
+          end
+
+          def quota_met?
+            counter == max_jobs_with_jitter
+          end
+
+          def quota_met_for_queue?(queue)
+            counter_for_queue(queue) == max_jobs_with_jitter_for_queue(queue)
           end
         end
 
@@ -121,13 +172,29 @@ module Sidekiq
             raise
           ensure
             if !exception_raised
-              self.class.mutex(queue).synchronize do
-                self.class.increment_counter!(queue)
+              self.class.mutex.synchronize do
+                terminate = false
+
+                # Increment the total counter
+                self.class.increment_counter!
 
-                if self.class.counter(queue) == self.class.max_jobs_with_jitter(queue)
+                # First check if the total quota has been met
+                if self.class.quota_met?
                   self.class.log_info("Max-Jobs quota met, shutting down pid: #{self.class.pid}")
-                  ::Process.kill('TERM', self.class.pid)
+                  terminate = true
+                end
+
+                # Increment the queue specific counter
+                self.class.increment_counter_for_queue!(queue)
+
+                # Now check if the queue specific quota has been met
+                if !terminate && self.class.quota_met_for_queue?(queue)
+                  self.class.log_info(%(Max-Jobs quota met for queue: "#{queue}", shutting down pid: #{self.class.pid}))
+                  terminate = true
                 end
+
+                # If applicable, TERMinate the `Process`
+                ::Process.kill('TERM', self.class.pid) if terminate
               end
             end
           end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-max-jobs
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Jonathan W. Zaleski
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-06-17 00:00:00.000000000 Z
+date: 2020-06-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq