sidekiq-ultimate 0.0.1.alpha.14 → 0.0.1.alpha.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4edc49fb50c61a2e76976baa800f742bd77792ae24f3f0bf6eca58483a0e9850
-  data.tar.gz: 743fca3a57ea28a216c7cff903cd2bd71cdb3955fe74d596a80659651aec6781
+  metadata.gz: f588d8dd927a0eaba4910ffbded5a8a24f69b6e8aae7c45b4bb28092373e6825
+  data.tar.gz: e2701097c8fbc5c99054bdb36a3bb4ae1089ee91abd1da74dfab4cab94f12ef7
 SHA512:
-  metadata.gz: 893c060ec92fe0d9fac453deb488b4d6f412bc4034cb25ed68cac3e0323a93ac28700c42e1f009c3636055977b6eaee68a779328bc957462cc17c342d984ff46
-  data.tar.gz: 2d8e07e65ad4b0ddb2e30a087364c0045ec7d03c3fb53c447289e48a15a339b83e5584b7abe5e7f518e4108c4469510effbddf24ae8009f81eb328f08d74b920
+  metadata.gz: f5a37773ca7013b24a5e7abb15aab91f7667f157b7e5d4bf43085c8268546a50d51319d351fad4db478428ee437d4acd43f762b33212a96d0d5d3bea57ad7d77
+  data.tar.gz: 7ceeb9436521f998f9f34e642f22aee1e8762e60c6ae9f5689b82902b95a27d6c1578aee89bd984ca81dca91fcfb4ef227eae170c9bfb1e854a1f17cd9ba08f6

data/lib/sidekiq/ultimate/expirable_set.rb

@@ -34,10 +34,15 @@ module Sidekiq
 
       # Create a new ExpirableSet instance.
       def initialize
-        @set = {}
+        @set = Hash.new(0.0)
         @mon = Monitor.new
       end
 
+      # Allow implicit coercion to Array:
+      #
+      #     ["x"] + ExpirableSet.new.add("y", :ttl => 10) # => ["x", "y"]
+      #
+      # @return [Array]
       alias to_ary to_a
 
       # Adds given element into the set.
@@ -50,9 +55,7 @@ module Sidekiq
           expires_at = Concurrent.monotonic_time + ttl
 
           # do not allow decrease element's expiry
-          break if @set[element] && @set[element] >= expires_at
-
-          @set[element] = expires_at
+          @set[element] = expires_at if @set[element] < expires_at
         end
 
         self
@@ -69,7 +72,7 @@ module Sidekiq
 
         @mon.synchronize do
           horizon = Concurrent.monotonic_time
-          @set.each { |k, v| v < horizon ? @set.delete(k) : yield(k) }
+          @set.each { |k, v| yield(k) if horizon <= v }
         end
 
         self

data/lib/sidekiq/ultimate/fetch.rb

@@ -11,12 +11,14 @@ module Sidekiq
   module Ultimate
     # Throttled reliable fetcher implementing reliable queue pattern.
     class Fetch
-      # Timeout to sleep between fetch retries in case of no job received.
+      # Delay between fetch retries in case of no job received.
       TIMEOUT = 2
 
-      # Timeout to sleep between queue fetch attempts in case if last job
-      # of it was throttled.
-      THROTTLE_TIMEOUT = 10
+      # Delay between queue poll attempts if last poll returned no jobs for it.
+      QUEUE_TIMEOUT = 5
+
+      # Delay between queue poll attempts if it's last job was throttled.
+      THROTTLE_TIMEOUT = 15
 
       def initialize(options)
         @exhausted = ExpirableSet.new
@@ -25,17 +27,25 @@ module Sidekiq
         @queues = options[:queues].map { |name| QueueName.new(name) }
 
         @queues.uniq! if @strict
+
+        @paused_queues            = []
+        @paused_queues_expires_at = 0
       end
 
+      # @return [UnitOfWork] if work can be processed
       def retrieve_work
         work = retrieve
 
-        return unless work
-        return work unless work.throttled?
+        if work&.throttled?
+          work.requeue_throttled
 
-        work.requeue_throttled
+          queue = QueueName.new(work.queue_name)
+          @exhausted.add(queue, :ttl => THROTTLE_TIMEOUT)
+
+          return nil
+        end
 
-        @exhausted.add(work.queue, :ttl => THROTTLE_TIMEOUT)
+        work
       end
 
       def self.bulk_requeue(units, _options)
@@ -55,7 +65,7 @@ module Sidekiq
             job = redis.rpoplpush(queue.pending, queue.inproc)
             return UnitOfWork.new(queue, job) if job
 
-            @exhausted.add(queue, :ttl => TIMEOUT)
+            @exhausted.add(queue, :ttl => QUEUE_TIMEOUT)
           end
         end
 
@@ -64,17 +74,21 @@ module Sidekiq
       end
 
       def queues
-        (@strict ? @queues : @queues.shuffle.uniq) - exhausted - paused_queues
-      end
+        queues = (@strict ? @queues : @queues.shuffle.uniq) - @exhausted.to_a
+
+        # Avoid calling heavier `paused_queue` if there's nothing to filter out
+        return queues if queues.empty?
 
-      def exhausted
-        @exhausted.to_a
+        queues - paused_queues
       end
 
       def paused_queues
-        Sidekiq::Throttled::QueuesPauser.instance.
-          instance_variable_get(:@paused_queues).
-          map { |q| QueueName[q] }
+        return @paused_queues if Time.now.to_i < @paused_queues_expires_at
+
+        @paused_queues.replace(Sidekiq::Throttled::QueuesPauser.instance.paused_queues.map { |q| QueueName[q] })
+        @paused_queues_expires_at = Time.now.to_i + 60
+
+        @paused_queues
       end
     end
   end

data/lib/sidekiq/ultimate/queue_name.rb

@@ -32,7 +32,7 @@ module Sidekiq
       # @param normalized [#to_s] Normalized (without any namespaces or `queue:`
       #   prefixes) queue name.
       # @param identity [#to_s] Sidekiq process identity.
-      def initialize(normalized, identity: self.class.process_identity)
+      def initialize(normalized, identity: Helper.identity)
         @normalized = -normalized.to_s
         @identity   = -identity.to_s
       end
@@ -113,10 +113,6 @@ module Sidekiq
       def self.[](name, **kwargs)
         new(name.to_s.sub(QUEUE_PREFIX_RE, "").freeze, **kwargs)
       end
-
-      def self.process_identity
-        Helper.identity
-      end
     end
   end
 end

data/lib/sidekiq/ultimate/resurrector.rb

@@ -95,7 +95,7 @@ module Sidekiq
               results  = redis.pipelined { |r| [r.time, r.get(LAST_RUN_KEY)] }
               distance = results[0][0] - results[1].to_i
 
-              return unless 60 < distance
+              break unless 60 < distance
 
               yield
 

data/lib/sidekiq/ultimate/version.rb

@@ -3,6 +3,6 @@
 module Sidekiq
   module Ultimate
     # Gem version.
-    VERSION = "0.0.1.alpha.14"
+    VERSION = "0.0.1.alpha.19"
   end
 end

data/sidekiq-ultimate.gemspec

@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
   # this gem instead.
   spec.add_runtime_dependency "sidekiq-throttled",  "~> 0.8"
 
-  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "bundler", "~> 2.0"
 
   spec.required_ruby_version = "~> 2.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-ultimate
 version: !ruby/object:Gem::Version
-  version: 0.0.1.alpha.14
+  version: 0.0.1.alpha.19
 platform: ruby
 authors:
 - Alexey Zapparov
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-11-27 00:00:00.000000000 Z
+date: 2020-10-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -86,14 +86,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '2.0'
 description: 
 email:
 - ixti@member.fsf.org
@@ -141,8 +141,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Sidekiq ultimate experience.