sidekiq-ultimate 0.0.1.alpha.16 → 0.0.1.alpha.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0a6a20e18e56f28bcbdeb1dcdba03a5978dce973551cd163262ceeb75c567435
-  data.tar.gz: c909bbef5ec8cbe2675347ad525c99774bcb068208d7d9f4fc091668cf8a6162
+  metadata.gz: f6f524ab4f87ba7a8c687046e2ef2893b29a709d35765620df2c212bb5325cc2
+  data.tar.gz: 7be36332d763fb27106eb4212a9454aa54b665df630ec27df90c777fc0c7638b
 SHA512:
-  metadata.gz: 8eea4a7d75983ab878774639dab1890bf61c233c05aa9572ad7aa69809b6aff002985057188a152b19f3f011e8b23b0468b2ee9d26e46a33f0378ca192773953
-  data.tar.gz: 1969844b087f9c57cf41dade645649a8c8e64be29e13ac76826a3ac9a6c144bde2231f4e8bd71fb83f9e8b47d6f34a3b77d31c95835dd844b944bd28b6802b32
+  metadata.gz: ce2c291e7b34d21adb5c9d1c57fb48208eb385714a9650233519f507d878ecc6efbb6089eb284e149ce262ea5e1b04e5f586628d5143bb42d4617e3e0729f53c
+  data.tar.gz: f4aca94bb0798554e6fcfda6617af4c9ec6f905ea31dcdceff1c8fb450523f9c423060630cba0d870b5ef03ad9b1ccf9fb19dda1c8587bfe9f1c35cbad8480c0

data/lib/sidekiq/ultimate/expirable_set.rb

@@ -4,8 +4,6 @@ require "monitor"
 
 require "concurrent/utility/monotonic_time"
 
-require "sidekiq/ultimate/debugging"
-
 module Sidekiq
   module Ultimate
     # List that tracks when elements were added and enumerates over those not
@@ -33,7 +31,6 @@ module Sidekiq
     # @private
     class ExpirableSet
       include Enumerable
-      include Debugging
 
       # Create a new ExpirableSet instance.
       def initialize
@@ -41,6 +38,11 @@ module Sidekiq
         @mon = Monitor.new
       end
 
+      # Allow implicit coercion to Array:
+      #
+      #     ["x"] + ExpirableSet.new.add("y", :ttl => 10) # => ["x", "y"]
+      #
+      # @return [Array]
       alias to_ary to_a
 
       # Adds given element into the set.
@@ -53,15 +55,9 @@ module Sidekiq
           expires_at = Concurrent.monotonic_time + ttl
 
           # do not allow decrease element's expiry
-          if @set[element] && @set[element] >= expires_at
-            debug! do
-              "#{element}'s expiry kept as is: #{@set[element]}; " \
-                "proposed expiry was: #{expires_at}"
-            end
-          else
-            @set[element] = expires_at
-            debug! { "#{element}'s expiry set to: #{expires_at}" }
-          end
+          break if @set.key?(element) && expires_at <= @set[element]
+
+          @set[element] = expires_at
         end
 
         self
@@ -78,10 +74,7 @@ module Sidekiq
 
         @mon.synchronize do
           horizon = Concurrent.monotonic_time
-
-          debug! { "Yielding elements above #{horizon} horizon" }
-
-          @set.each { |k, v| v < horizon ? @set.delete(k) : yield(k) }
+          @set.each { |k, v| yield(k) if horizon <= v }
         end
 
         self

data/lib/sidekiq/ultimate/fetch.rb

@@ -2,7 +2,6 @@
 
 require "sidekiq/throttled"
 
-require "sidekiq/ultimate/debugging"
 require "sidekiq/ultimate/expirable_set"
 require "sidekiq/ultimate/queue_name"
 require "sidekiq/ultimate/resurrector"
@@ -12,33 +11,32 @@ module Sidekiq
   module Ultimate
     # Throttled reliable fetcher implementing reliable queue pattern.
     class Fetch
-      include Debugging
-
-      # Timeout to sleep between fetch retries in case of no job received.
+      # Delay between fetch retries in case of no job received.
       TIMEOUT = 2
 
+      # Delay between queue poll attempts if last poll returned no jobs for it.
       QUEUE_TIMEOUT = 5
 
-      # Timeout to sleep between queue fetch attempts in case if last job
-      # of it was throttled.
+      # Delay between queue poll attempts if it's last job was throttled.
       THROTTLE_TIMEOUT = 15
 
       def initialize(options)
         @exhausted = ExpirableSet.new
 
+        @debug  = ENV["DEBUG_SIDEKIQ_ULTIMATE"] ? {} : nil
         @strict = options[:strict] ? true : false
         @queues = options[:queues].map { |name| QueueName.new(name) }
 
         @queues.uniq! if @strict
       end
 
+      # @return [UnitOfWork] if work can be processed
       def retrieve_work
         work = retrieve
 
         if work&.throttled?
           work.requeue_throttled
 
-          debug! { "Queue #{work.queue} got throttled job." }
           @exhausted.add(work.queue, :ttl => THROTTLE_TIMEOUT)
 
           return nil
@@ -61,28 +59,26 @@ module Sidekiq
       def retrieve
         Sidekiq.redis do |redis|
           queues.each do |queue|
+            debug!(queue)
+
             job = redis.rpoplpush(queue.pending, queue.inproc)
             return UnitOfWork.new(queue, job) if job
 
-            debug! { "Queue #{queue} has no job." }
             @exhausted.add(queue, :ttl => QUEUE_TIMEOUT)
           end
         end
 
-        debug! { "No jobs in any queues." }
-
         sleep TIMEOUT
         nil
       end
 
       def queues
-        queues  = @strict ? @queues : @queues.shuffle.uniq
-        queues -= @exhausted_queues.to_a
-        queues -= paused_queues unless queues.empty?
+        queues = (@strict ? @queues : @queues.shuffle.uniq) - @exhausted.to_a
 
-        debug! { "Queues to poll: #{queues.map(&:to_s).join(', ')}" }
+        # Avoid calling heavier `paused_queue` if there's nothing to filter out
+        return queues if queues.empty?
 
-        queues
+        queues - paused_queues
       end
 
       def paused_queues
@@ -90,6 +86,18 @@ module Sidekiq
           instance_variable_get(:@paused_queues).
           map { |q| QueueName[q] }
       end
+
+      def debug!(queue)
+        return unless @debug
+
+        previous, @debug[queue] = @debug[queue], Concurrent.monotonic_time
+
+        return unless previous
+
+        Sidekiq.logger.debug do
+          "Queue #{queue} last time polled: #{@debug[queue] - previous} s ago"
+        end
+      end
     end
   end
 end

data/lib/sidekiq/ultimate/fetch/debug.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "sidekiq/throttled"
+
+require "sidekiq/ultimate/debugging"
+require "sidekiq/ultimate/expirable_set"
+require "sidekiq/ultimate/queue_name"
+require "sidekiq/ultimate/resurrector"
+require "sidekiq/ultimate/unit_of_work"
+
+module Sidekiq
+  module Ultimate
+    # Throttled reliable fetcher implementing reliable queue pattern.
+    class Fetch
+      class Debug
+        def initialize
+          @queues_stats = {}
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/ultimate/version.rb

@@ -3,6 +3,6 @@
 module Sidekiq
   module Ultimate
     # Gem version.
-    VERSION = "0.0.1.alpha.16"
+    VERSION = "0.0.1.alpha.17"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-ultimate
 version: !ruby/object:Gem::Version
-  version: 0.0.1.alpha.16
+  version: 0.0.1.alpha.17
 platform: ruby
 authors:
 - Alexey Zapparov
@@ -112,9 +112,9 @@ files:
 - README.md
 - Rakefile
 - lib/sidekiq/ultimate.rb
-- lib/sidekiq/ultimate/debugging.rb
 - lib/sidekiq/ultimate/expirable_set.rb
 - lib/sidekiq/ultimate/fetch.rb
+- lib/sidekiq/ultimate/fetch/debug.rb
 - lib/sidekiq/ultimate/queue_name.rb
 - lib/sidekiq/ultimate/resurrector.rb
 - lib/sidekiq/ultimate/resurrector/resurrect.lua

data/lib/sidekiq/ultimate/debugging.rb

@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-module Sidekiq
-  module Ultimate
-    module Debugging
-      private
-
-      def debug!
-        Sidekiq.logger.debug { "[#{self.class}] #{yield}" }
-      end
-    end
-  end
-end