active_publisher 1.0.3-java → 1.1.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c842c3825c85ddd417ae7b44992f0dd116e04952
-  data.tar.gz: 52903e7be84dc6295e68446fdb80e1d6cf898421
+  metadata.gz: bbb7f6d380395ad17ca81b0128f6ff2cec2e0e98
+  data.tar.gz: 6ce766a9e768c7adc0834f94dd10473e73a8a144
 SHA512:
-  metadata.gz: 7a5e4ec82f01a97f3c904ebaf1f3501a45857a072eb245f24a1a717479483cd4a452d8338beddb9d05bf786164d076a185270daa47a803c2ee36e4efe4598cd6
-  data.tar.gz: 64521da6574d628d62a26fa98bcbfedf87ac54ddeca47fd9a791b3eaed545fcc867d4861ff6f0f509cfcc9e6f8dd327ac20854bc7321bfbced039ce7dbbbebb8
+  metadata.gz: ad04a249d94c9cb8fc91b5c195ddab86e3a6788e0512c9101d6652f0b7e3d5438b100951c50dc9f3c1406c24a3e847168cd6057ae5ba1ea76b0adb1841e084c5
+  data.tar.gz: d10b6b66a2239e5076ef55e78dcac184d9937c0f114423a1399adce9812b49f9cbd1428d7a7aa46fe9ac63a4e309c1e2bd23e4a0ef5588d438391a90f6ace209

data/README.md

@@ -51,6 +51,8 @@ Defaults for the configuration are:
   :heartbeat => 5,
   :host => "localhost",
   :hosts => [],
+  :max_async_publisher_lag_time => 10,
+  :network_recovery_interval => 1,
   :password => "guest",
   :port => 5672,
   :publisher_confirms => false,
@@ -110,4 +112,3 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/[USERN
 ## License
 
 The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
-

data/active_publisher.gemspec

@@ -27,7 +27,8 @@ Gem::Specification.new do |spec|
   end
 
   spec.add_dependency 'activesupport', '>= 3.2'
-  spec.add_dependency 'multi_op_queue', '>= 0.1.2'
+  spec.add_dependency 'concurrent-ruby'
+  spec.add_dependency 'multi_op_queue', '>= 0.2.0'
 
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "pry"

data/lib/active_publisher/async/in_memory_adapter/async_queue.rb

@@ -60,20 +60,25 @@ module ActivePublisher
 
         def create_and_supervise_consumer!
           @consumer = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
-          @supervisor = ::Thread.new do
-            loop do
-              unless consumer.alive?
-                consumer.kill
-                @consumer = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
-              end
 
-              # Notify the current queue size.
-              ::ActiveSupport::Notifications.instrument "async_queue_size.active_publisher", queue.size
+          supervisor_task = ::Concurrent::TimerTask.new(:execution_interval => supervisor_interval) do
+            current_time = ::Time.now
+
+            # Consumer is lagging if it does not "tick" at least once every 10 seconds.
+            seconds_since_last_tick = current_time - consumer.last_tick_at
+            consumer_is_lagging = seconds_since_last_tick > ::ActivePublisher.configuration.max_async_publisher_lag_time
+            logger.error "ActivePublisher consumer is lagging. Last consumer tick was #{seconds_since_last_tick} seconds ago." if consumer_is_lagging
 
-              # Pause before checking the consumer again.
-              sleep supervisor_interval
+            # Check to see if we should restart the consumer.
+            if !consumer.alive? || consumer_is_lagging
+              consumer.kill
+              @consumer = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
             end
+
+            # Notify the current queue size.
+            ::ActiveSupport::Notifications.instrument "async_queue_size.active_publisher", queue.size
           end
+          supervisor_task.execute
         end
       end
 

data/lib/active_publisher/async/in_memory_adapter/consumer_thread.rb

@@ -2,17 +2,27 @@ module ActivePublisher
   module Async
     module InMemoryAdapter
       class ConsumerThread
-        attr_reader :thread, :queue, :sampled_queue_size
+        attr_reader :thread, :queue, :sampled_queue_size, :last_tick_at
 
         if ::RUBY_PLATFORM == "java"
-          NETWORK_ERRORS = [::MarchHare::Exception, ::Java::ComRabbitmqClient::AlreadyClosedException, ::Java::JavaIo::IOException].freeze
+          NETWORK_ERRORS = [::MarchHare::NetworkException, ::MarchHare::ConnectionRefused,
+                            ::Java::ComRabbitmqClient::AlreadyClosedException, ::Java::JavaIo::IOException].freeze
         else
-          NETWORK_ERRORS = [::Bunny::Exception, ::Timeout::Error, ::IOError].freeze
+          NETWORK_ERRORS = [::Bunny::NetworkFailure, ::Bunny::TCPConnectionFailed, ::Bunny::ConnectionTimeout,
+                            ::Timeout::Error, ::IOError].freeze
+        end
+
+        if ::RUBY_PLATFORM == "java"
+          PRECONDITION_ERRORS = [::MarchHare::PreconditionFailed]
+        else
+          PRECONDITION_ERRORS = [::Bunny::PreconditionFailed]
         end
 
         def initialize(listen_queue)
           @queue = listen_queue
           @sampled_queue_size = queue.size
+
+          update_last_tick_at
           start_thread
         end
 
@@ -41,38 +51,46 @@ module ActivePublisher
           channel
         end
 
+        def update_last_tick_at
+          @last_tick_at = ::Time.now
+        end
+
         def start_thread
           return if alive?
           @thread = ::Thread.new do
             loop do
               # Sample the queue size so we don't shutdown when messages are in flight.
               @sampled_queue_size = queue.size
-              current_messages = queue.pop_up_to(50)
+              current_messages = queue.pop_up_to(50, :timeout => 0.1)
+              update_last_tick_at
+              # If the queue is empty, we should continue to update to "last_tick_at" time.
+              next if current_messages.nil?
+
+              # We only look at active publisher messages. Everything else is dropped.
+              current_messages.select! { |message| message.is_a?(::ActivePublisher::Message) }
 
               begin
                 @channel ||= make_channel
 
                 # Only open a single connection for each group of messages to an exchange
                 current_messages.group_by(&:exchange_name).each do |exchange_name, messages|
-                  begin
-                    current_messages -= messages
-                    publish_all(@channel, exchange_name, messages)
-                  ensure
-                    current_messages.concat(messages)
-                  end
+                  publish_all(@channel, exchange_name, messages)
+                  current_messages -= messages
                 end
               rescue *NETWORK_ERRORS
                 # Sleep because connection is down
                 await_network_reconnect
-
-                # Requeue and try again.
-                queue.concat(current_messages)
               rescue => unknown_error
                 ::ActivePublisher.configuration.error_handler.call(unknown_error, {:number_of_messages => current_messages.size})
                 current_messages.each do |message|
                   # Degrade to single message publish ... or at least attempt to
                   begin
                     ::ActivePublisher.publish(message.route, message.payload, message.exchange_name, message.options)
+                    current_messages.delete(message)
+                  rescue *PRECONDITION_ERRORS => error
+                    # Delete messages if rabbitmq cannot declare the exchange (or somet other precondition failed).
+                    ::ActivePublisher.configuration.error_handler.call(error, {:reason => "precondition failed", :message => message})
+                    current_messages.delete(message)
                   rescue => individual_error
                     ::ActivePublisher.configuration.error_handler.call(individual_error, {:route => message.route, :payload => message.payload, :exchange_name => message.exchange_name, :options => message.options})
                   end
@@ -81,6 +99,9 @@ module ActivePublisher
                 # TODO: Find a way to bubble this out of the thread for logging purposes.
                 # Reraise the error out of the publisher loop. The Supervisor will restart the consumer.
                 raise unknown_error
+              ensure
+                # Always requeue anything that gets stuck.
+                queue.concat(current_messages) unless current_messages.nil?
               end
             end
           end
@@ -89,37 +110,23 @@ module ActivePublisher
         def publish_all(channel, exchange_name, messages)
           ::ActiveSupport::Notifications.instrument "message_published.active_publisher", :message_count => messages.size do
             exchange = channel.topic(exchange_name)
-            potentially_retry = []
-            loop do
-              break if messages.empty?
-              message = messages.shift
-
-              fail ActivePublisher::UnknownMessageClassError, "bulk publish messages must be ActivePublisher::Message" unless message.is_a?(ActivePublisher::Message)
+            messages.each do |message|
               fail ActivePublisher::ExchangeMismatchError, "bulk publish messages must match publish_all exchange_name" if message.exchange_name != exchange_name
 
-              begin
-                options = ::ActivePublisher.publishing_options(message.route, message.options || {})
-                exchange.publish(message.payload, options)
-                potentially_retry << message
-              rescue
-                messages << message
-                raise
-              end
+              options = ::ActivePublisher.publishing_options(message.route, message.options || {})
+              exchange.publish(message.payload, options)
             end
-            wait_for_confirms(channel, messages, potentially_retry)
+            wait_for_confirms(channel)
           end
         end
 
-        def wait_for_confirms(channel, messages, potentially_retry)
+        def wait_for_confirms(channel)
           return true unless channel.using_publisher_confirms?
           if channel.method(:wait_for_confirms).arity > 0
             channel.wait_for_confirms(::ActivePublisher.configuration.publisher_confirms_timeout)
           else
             channel.wait_for_confirms
           end
-        rescue
-          messages.concat(potentially_retry)
-          raise
         end
       end
     end

data/lib/active_publisher/async/in_memory_adapter.rb

@@ -1,6 +1,7 @@
 require "active_publisher/message"
 require "active_publisher/async/in_memory_adapter/async_queue"
 require "active_publisher/async/in_memory_adapter/consumer_thread"
+require "concurrent"
 require "multi_op_queue"
 
 module ActivePublisher

data/lib/active_publisher/configuration.rb

@@ -6,6 +6,7 @@ module ActivePublisher
                   :heartbeat,
                   :host,
                   :hosts,
+                  :max_async_publisher_lag_time,
                   :network_recovery_interval,
                   :password,
                   :port,
@@ -34,6 +35,7 @@ module ActivePublisher
       :host => "localhost",
       :hosts => [],
       :password => "guest",
+      :max_async_publisher_lag_time => 10,
       :network_recovery_interval => NETWORK_RECOVERY_INTERVAL,
       :port => 5672,
       :publisher_confirms => false,

data/lib/active_publisher/version.rb

@@ -1,3 +1,3 @@
 module ActivePublisher
-  VERSION = "1.0.3"
+  VERSION = "1.1.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: active_publisher
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.1.0
 platform: java
 authors:
 - Brian Stien
@@ -12,7 +12,7 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2017-12-08 00:00:00.000000000 Z
+date: 2018-01-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -47,7 +47,21 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.2
+        version: '0'
+  name: concurrent-ruby
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.0
   name: multi_op_queue
   prerelease: false
   type: :runtime
@@ -55,7 +69,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.2
+        version: 0.2.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements: