active_publisher 1.2.6-java → 1.3.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5eb0ac3e4acd1a5a7cd668d9eef3bd900af1964781b65eb22c3b97d17add28b4
-  data.tar.gz: 8bb17f982325ef1d4923939f4e13729affddfc4320c793305837d9a368c51f22
+  metadata.gz: 7447a8948c2a4e24769ef9733ab12e843266564385535c2cb7bfbb8d485ec50a
+  data.tar.gz: c8fef7d08eef182420cbdef0fa847b40cc5f9e14ef96b81ad076fcb885526717
 SHA512:
-  metadata.gz: 8ce688832579a6ac3eb94051b43b5fd956c10bf8c1e64b1a7354c806543674810a6784f18caab97165b0115cc52e9aa156732f04cd2be93bd69e36bf38eb9028
-  data.tar.gz: 0fd321640fe284276f11c5e262df01f9731e1ea9a870cddc22a85fee6753663ee315fc427c2d1aa0ecab29afc35c31e9cefbd14283db4491cb16fb11398ddf3e
+  metadata.gz: 6e5f3d51840ac76b600444eb36e4b358f4fed9ae5ae876eed27747f8e89f67273d7cf52fc608dc788f0631257b6f4c6d5e05b9f014a05b39175958f1956fefa1
+  data.tar.gz: e965f7bedbc7449c3cb6f6e86afa718be8d28853ba46acec02be6589eebf58389c276b4aa3c90b8a2a919825c17acfd22e93987d4212a4cec535219e9d34158f

data/README.md

@@ -55,7 +55,9 @@ Defaults for the configuration are:
   :network_recovery_interval => 1,
   :password => "guest",
   :port => 5672,
+  :publisher_threads => 1,
   :publisher_confirms => false,
+  :publisher_confirms_timeout => 5_000,
   :seconds_to_wait_for_graceful_shutdown => 30,
   :timeout => 1,
   :tls => false,

data/lib/active_publisher/async/in_memory_adapter/async_queue.rb

@@ -14,14 +14,15 @@ module ActivePublisher
           :max_queue_size,
           :supervisor_interval
 
-        attr_reader :consumer, :queue, :supervisor
+        attr_reader :consumers, :queue, :supervisor
 
         def initialize(back_pressure_strategy, max_queue_size, supervisor_interval)
           self.back_pressure_strategy = back_pressure_strategy
           @max_queue_size = max_queue_size
           @supervisor_interval = supervisor_interval
           @queue = ::MultiOpQueue::Queue.new
-          create_and_supervise_consumer!
+          @consumers = {}
+          create_and_supervise_consumers!
         end
 
         def back_pressure_strategy=(strategy)
@@ -52,36 +53,44 @@ module ActivePublisher
         def size
           # Requests might be in flight (out of the queue, but not yet published), so taking the max should be
           # good enough to make sure we're honest about the actual queue size.
-          return queue.size if consumer.nil?
-          [queue.size, consumer.sampled_queue_size].max
+          return queue.size if consumers.empty?
+          [queue.size, consumer_sampled_queue_size].max
         end
 
         private
 
-        def create_and_supervise_consumer!
-          @consumer = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
+        def create_and_supervise_consumers!
+          ::ActivePublisher.configuration.publisher_threads.times do
+            consumer_id = ::SecureRandom.uuid
+            consumers[consumer_id] = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
+            supervisor_task = ::Concurrent::TimerTask.new(:execution_interval => supervisor_interval) do
+              current_time = ::Time.now
+              consumer = consumers[consumer_id]
 
-          supervisor_task = ::Concurrent::TimerTask.new(:execution_interval => supervisor_interval) do
-            current_time = ::Time.now
+              # Consumer is lagging if it does not "tick" at least once every 10 seconds.
+              seconds_since_last_tick = current_time - consumer.last_tick_at
+              consumer_is_lagging = seconds_since_last_tick > ::ActivePublisher.configuration.max_async_publisher_lag_time
+              logger.error "ActivePublisher consumer is lagging. Last consumer tick was #{seconds_since_last_tick} seconds ago." if consumer_is_lagging
 
-            # Consumer is lagging if it does not "tick" at least once every 10 seconds.
-            seconds_since_last_tick = current_time - consumer.last_tick_at
-            consumer_is_lagging = seconds_since_last_tick > ::ActivePublisher.configuration.max_async_publisher_lag_time
-            logger.error "ActivePublisher consumer is lagging. Last consumer tick was #{seconds_since_last_tick} seconds ago." if consumer_is_lagging
+              # Check to see if we should restart the consumer.
+              if !consumer.alive? || consumer_is_lagging
+                consumer.kill rescue nil
+                consumers[consumer_id] = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
+                ::ActiveSupport::Notifications.instrument "async_queue.thread_restart"
+              end
 
-            # Check to see if we should restart the consumer.
-            if !consumer.alive? || consumer_is_lagging
-              consumer.kill rescue nil
-              @consumer = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
+              # Notify the current queue size.
+              ::ActiveSupport::Notifications.instrument "async_queue_size.active_publisher", queue.size
             end
-
-            # Notify the current queue size.
-            ::ActiveSupport::Notifications.instrument "async_queue_size.active_publisher", queue.size
+            supervisor_task.execute
           end
-          supervisor_task.execute
         end
-      end
 
+        def consumer_sampled_queue_size
+          consumers.values.map(&:sampled_queue_size).max
+        end
+
+      end
     end
   end
 end

data/lib/active_publisher/async/redis_adapter/consumer.rb

@@ -9,28 +9,33 @@ module ActivePublisher
           :timeout_interval => 5, # seconds
         }
 
-        attr_reader :consumer, :queue, :supervisor
+        attr_reader :consumers, :queue, :supervisor
 
         def initialize(redis_pool)
           @queue = ::ActivePublisher::Async::RedisAdapter::RedisMultiPopQueue.new(redis_pool, ::ActivePublisher::Async::RedisAdapter::REDIS_LIST_KEY)
-          create_and_supervise_consumer!
+          @consumers = {}
+          create_and_supervise_consumers!
         end
 
-        def create_and_supervise_consumer!
-          @consumer = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
-
-          supervisor_task = ::Concurrent::TimerTask.new(SUPERVISOR_INTERVAL) do
-            # This may also be the place to start additional publishers when we are getting backed up ... ?
-            unless consumer.alive?
-              consumer.kill rescue nil
-              @consumer = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
+        def create_and_supervise_consumers!
+          ::ActivePublisher.configuration.publisher_threads.times do
+            consumer_id = ::SecureRandom.uuid
+            consumers[consumer_id] = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
+
+            supervisor_task = ::Concurrent::TimerTask.new(SUPERVISOR_INTERVAL) do
+              consumer = consumers[consumer_id]
+              unless consumer.alive?
+                consumer.kill rescue nil
+                consumers[consumer_id] = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
+                ::ActiveSupport::Notifications.instrument "async_queue.thread_restart"
+              end
+
+              # Notify the current queue size.
+              ::ActiveSupport::Notifications.instrument "redis_async_queue_size.active_publisher", queue.size
             end
 
-            # Notify the current queue size.
-            ::ActiveSupport::Notifications.instrument "redis_async_queue_size.active_publisher", queue.size
+            supervisor_task.execute
           end
-
-          supervisor_task.execute
         end
 
         def size

data/lib/active_publisher/configuration.rb

@@ -12,6 +12,7 @@ module ActivePublisher
                   :network_recovery_interval,
                   :password,
                   :port,
+                  :publisher_threads,
                   :publisher_confirms,
                   :publisher_confirms_timeout,
                   :seconds_to_wait_for_graceful_shutdown,
@@ -41,6 +42,7 @@ module ActivePublisher
       :max_async_publisher_lag_time => 10,
       :network_recovery_interval => NETWORK_RECOVERY_INTERVAL,
       :port => 5672,
+      :publisher_threads => 1,
       :publisher_confirms => false,
       :publisher_confirms_timeout => 5_000, #specified as a number of milliseconds
       :seconds_to_wait_for_graceful_shutdown => 30,

data/lib/active_publisher/version.rb

@@ -1,3 +1,3 @@
 module ActivePublisher
-  VERSION = "1.2.6"
+  VERSION = "1.3.0"
 end

data/lib/active_publisher.rb

@@ -4,6 +4,7 @@ else
   require "bunny"
 end
 require "active_support"
+require "securerandom"
 require "thread"
 
 require "active_publisher/logging"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: active_publisher
 version: !ruby/object:Gem::Version
-  version: 1.2.6
+  version: 1.3.0
 platform: java
 authors:
 - Brian Stien
@@ -12,7 +12,7 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-09-29 00:00:00.000000000 Z
+date: 2021-10-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -21,8 +21,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.7'
   name: march_hare
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -35,8 +35,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '3.2'
   name: activesupport
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -49,8 +49,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: concurrent-ruby
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -63,8 +63,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.2.0
   name: multi_op_queue
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -77,8 +77,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: benchmark-ips
-  type: :development
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -91,8 +91,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: bundler
-  type: :development
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -105,8 +105,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: connection_pool
-  type: :development
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -119,8 +119,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: fakeredis
-  type: :development
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -133,8 +133,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: pry
-  type: :development
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -147,8 +147,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: rake
-  type: :development
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -161,8 +161,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '3.2'
   name: rspec
-  type: :development
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -223,7 +223,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
+rubygems_version: 3.2.28
 signing_key:
 specification_version: 4
 summary: Aims to make publishing work across MRI and jRuby painless and add some nice