sidekiq-ultimate 0.0.1.alpha.14 → 0.0.1.alpha.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4edc49fb50c61a2e76976baa800f742bd77792ae24f3f0bf6eca58483a0e9850
-  data.tar.gz: 743fca3a57ea28a216c7cff903cd2bd71cdb3955fe74d596a80659651aec6781
+  metadata.gz: 441f5bf063360de121a2c0f33c4dffe39b2313e841830c786b21812649b465b8
+  data.tar.gz: 3a5304085501c4375af4b95c1dfdfd26fe06cbb56756ab55afab0004c97a22ec
 SHA512:
-  metadata.gz: 893c060ec92fe0d9fac453deb488b4d6f412bc4034cb25ed68cac3e0323a93ac28700c42e1f009c3636055977b6eaee68a779328bc957462cc17c342d984ff46
-  data.tar.gz: 2d8e07e65ad4b0ddb2e30a087364c0045ec7d03c3fb53c447289e48a15a339b83e5584b7abe5e7f518e4108c4469510effbddf24ae8009f81eb328f08d74b920
+  metadata.gz: df918771242ecaf8c3bab9dfc47cb2992a6bbb0dc1c6ab141c1af1cdb86d5cc10134f95a6ec1d7b735136f718116140037a57b51f50e6b90f8e8adc23012469c
+  data.tar.gz: 89a4316a4b869a7758f488575918a3ddb3521c93c9d22f9fc20beaa71350c36e0c8feb268b03c98d2c6f4731c3d8711b9f700ef70a709a2d98c583a4a334aaff

data/lib/sidekiq/ultimate/debugging.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Ultimate
+    module Debugging
+      private
+
+      def debug!
+        return unless ENV.key? "DEBUG_SIDEKIQ_ULTIMATE"
+        Sidekiq.logger.debug { "[#{self.class}] #{yield}" }
+      end
+    end
+  end
+end

data/lib/sidekiq/ultimate/expirable_set.rb

@@ -4,6 +4,8 @@ require "monitor"
 
 require "concurrent/utility/monotonic_time"
 
+require "sidekiq/ultimate/debugging"
+
 module Sidekiq
   module Ultimate
     # List that tracks when elements were added and enumerates over those not
@@ -31,6 +33,7 @@ module Sidekiq
     # @private
     class ExpirableSet
       include Enumerable
+      include Debugging
 
       # Create a new ExpirableSet instance.
       def initialize
@@ -50,9 +53,15 @@ module Sidekiq
           expires_at = Concurrent.monotonic_time + ttl
 
           # do not allow decrease element's expiry
-          break if @set[element] && @set[element] >= expires_at
-
-          @set[element] = expires_at
+          if @set[element] && @set[element] >= expires_at
+            debug! do
+              "#{element}'s expiry kept as is: #{@set[element]}; " \
+                "proposed expiry was: #{expires_at}"
+            end
+          else
+            @set[element] = expires_at
+            debug! { "#{element}'s expiry set to: #{expires_at}" }
+          end
         end
 
         self
@@ -69,6 +78,9 @@ module Sidekiq
 
         @mon.synchronize do
           horizon = Concurrent.monotonic_time
+
+          debug! { "Yielding elements above #{horizon} horizon" }
+
           @set.each { |k, v| v < horizon ? @set.delete(k) : yield(k) }
         end
 

data/lib/sidekiq/ultimate/fetch.rb

@@ -2,6 +2,7 @@
 
 require "sidekiq/throttled"
 
+require "sidekiq/ultimate/debugging"
 require "sidekiq/ultimate/expirable_set"
 require "sidekiq/ultimate/queue_name"
 require "sidekiq/ultimate/resurrector"
@@ -11,12 +12,16 @@ module Sidekiq
   module Ultimate
     # Throttled reliable fetcher implementing reliable queue pattern.
     class Fetch
+      include Debugging
+
       # Timeout to sleep between fetch retries in case of no job received.
       TIMEOUT = 2
 
+      QUEUE_TIMEOUT = 5
+
       # Timeout to sleep between queue fetch attempts in case if last job
       # of it was throttled.
-      THROTTLE_TIMEOUT = 10
+      THROTTLE_TIMEOUT = 15
 
       def initialize(options)
         @exhausted = ExpirableSet.new
@@ -30,12 +35,16 @@ module Sidekiq
       def retrieve_work
         work = retrieve
 
-        return unless work
-        return work unless work.throttled?
+        if work&.throttled?
+          work.requeue_throttled
 
-        work.requeue_throttled
+          debug! { "Queue #{queue} got throttled job." }
+          @exhausted.add(work.queue, :ttl => THROTTLE_TIMEOUT)
+
+          return nil
+        end
 
-        @exhausted.add(work.queue, :ttl => THROTTLE_TIMEOUT)
+        work
       end
 
       def self.bulk_requeue(units, _options)
@@ -55,20 +64,25 @@ module Sidekiq
             job = redis.rpoplpush(queue.pending, queue.inproc)
             return UnitOfWork.new(queue, job) if job
 
-            @exhausted.add(queue, :ttl => TIMEOUT)
+            debug! { "Queue #{queue} has no job." }
+            @exhausted.add(queue, :ttl => QUEUE_TIMEOUT)
           end
         end
 
+        debug! { "No jobs in any queues." }
+
         sleep TIMEOUT
         nil
       end
 
       def queues
-        (@strict ? @queues : @queues.shuffle.uniq) - exhausted - paused_queues
-      end
+        queues  = @strict ? @queues : @queues.shuffle.uniq
+        queues -= @exhausted_queues.to_a
+        queues -= paused_queues unless queues.empty?
+
+        debug! { "Queues to poll: #{queues.map(&:to_s).join(', ')}" }
 
-      def exhausted
-        @exhausted.to_a
+        queues
       end
 
       def paused_queues

data/lib/sidekiq/ultimate/queue_name.rb

@@ -32,7 +32,7 @@ module Sidekiq
       # @param normalized [#to_s] Normalized (without any namespaces or `queue:`
       #   prefixes) queue name.
       # @param identity [#to_s] Sidekiq process identity.
-      def initialize(normalized, identity: self.class.process_identity)
+      def initialize(normalized, identity: Helper.identity)
         @normalized = -normalized.to_s
         @identity   = -identity.to_s
       end
@@ -113,10 +113,6 @@ module Sidekiq
       def self.[](name, **kwargs)
         new(name.to_s.sub(QUEUE_PREFIX_RE, "").freeze, **kwargs)
       end
-
-      def self.process_identity
-        Helper.identity
-      end
     end
   end
 end

data/lib/sidekiq/ultimate/resurrector.rb

@@ -95,7 +95,7 @@ module Sidekiq
               results  = redis.pipelined { |r| [r.time, r.get(LAST_RUN_KEY)] }
               distance = results[0][0] - results[1].to_i
 
-              return unless 60 < distance
+              break unless 60 < distance
 
               yield
 

data/lib/sidekiq/ultimate/version.rb

@@ -3,6 +3,6 @@
 module Sidekiq
   module Ultimate
     # Gem version.
-    VERSION = "0.0.1.alpha.14"
+    VERSION = "0.0.1.alpha.15"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-ultimate
 version: !ruby/object:Gem::Version
-  version: 0.0.1.alpha.14
+  version: 0.0.1.alpha.15
 platform: ruby
 authors:
 - Alexey Zapparov
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-11-27 00:00:00.000000000 Z
+date: 2019-07-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -112,6 +112,7 @@ files:
 - README.md
 - Rakefile
 - lib/sidekiq/ultimate.rb
+- lib/sidekiq/ultimate/debugging.rb
 - lib/sidekiq/ultimate/expirable_set.rb
 - lib/sidekiq/ultimate/fetch.rb
 - lib/sidekiq/ultimate/queue_name.rb
@@ -141,8 +142,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Sidekiq ultimate experience.