active_publisher 1.2.5-java → 1.3.0.pre0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: 7a878924c71e32c34435e80052a8f241b357d2f7b70f3582f7b8920b942e63c0
-  data.tar.gz: 60b4eb97d20e389ce4865ef393d69a7d49bdac2e47a3aa26eb6e7ca32e459335
+SHA1:
+  metadata.gz: 8ab5bdf4352d3913b934a0e54a0085effe6fcf69
+  data.tar.gz: 4baa705ae3174699cd4d55ecd51b907dd14ce621
 SHA512:
-  metadata.gz: 8f6399701888401993164b31ccb4fff4f14eb13414a7586d7baacac0c49f2c876dfe33035ce091ec4b62ee4a5b64ad961a7deab79691098401388153404c724b
-  data.tar.gz: 2a1d1347293457b9e5b663d7f4815a176e8d9afdc80e472893c4f1f5206bdfc057bfb6d692067924727aac1e3a6f42bc2fff20cd13698bca12be61e1b72d84bc
+  metadata.gz: c47d2ae4a05f959983fe022c6aa2face66684a86b8d9bc3e923f55dcdbdcb826b75c21d1c719ad4ace2e9c3cf2ea2b3aa7d4f43a5e9afd2216399fee7359e5b8
+  data.tar.gz: 7930ca08472a9370f6a39789a0d845b19553c0d4ebc98d31f28202182e34a5333a0fe2d5c62678e7972d25a5fc06e60ffffa64d4c2fb939362dba72ba430e89c

data/.travis.yml

@@ -0,0 +1,9 @@
+language: ruby
+rvm:
+ - 2.3
+ - 2.4
+ - jruby-9.1.12.0
+services:
+ - rabbitmq
+sudo: false
+cache: bundler

data/README.md

@@ -55,7 +55,9 @@ Defaults for the configuration are:
   :network_recovery_interval => 1,
   :password => "guest",
   :port => 5672,
+  :publisher_threads => 1,
   :publisher_confirms => false,
+  :publisher_confirms_timeout => 5_000,
   :seconds_to_wait_for_graceful_shutdown => 30,
   :timeout => 1,
   :tls => false,

data/active_publisher.gemspec

@@ -35,6 +35,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "connection_pool"
   spec.add_development_dependency "fakeredis"
   spec.add_development_dependency "pry"
-  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.2"
 end

data/lib/active_publisher.rb

@@ -4,6 +4,7 @@ else
   require "bunny"
 end
 require "active_support"
+require "securerandom"
 require "thread"
 
 require "active_publisher/logging"
@@ -35,7 +36,7 @@ module ActivePublisher
   # @param [Hash] options hash to set message parameters (e.g. headers)
   def self.publish(route, payload, exchange_name, options = {})
     with_exchange(exchange_name) do |exchange|
-      ::ActiveSupport::Notifications.instrument "message_published.active_publisher", :route => route do
+      ::ActiveSupport::Notifications.instrument "message_published.active_publisher" do
         exchange.publish(payload, publishing_options(route, options))
       end
     end
@@ -51,7 +52,7 @@ module ActivePublisher
         fail ActivePublisher::ExchangeMismatchError, "bulk publish messages must match publish_all exchange_name" if message.exchange_name != exchange_name
 
         begin
-          ::ActiveSupport::Notifications.instrument "message_published.active_publisher", :route => message.route do
+          ::ActiveSupport::Notifications.instrument "message_published.active_publisher" do
             exchange.publish(message.payload, publishing_options(message.route, message.options || {}))
           end
         rescue

data/lib/active_publisher/async/in_memory_adapter/async_queue.rb

@@ -14,14 +14,15 @@ module ActivePublisher
           :max_queue_size,
           :supervisor_interval
 
-        attr_reader :consumer, :queue, :supervisor
+        attr_reader :consumers, :queue, :supervisor
 
         def initialize(back_pressure_strategy, max_queue_size, supervisor_interval)
           self.back_pressure_strategy = back_pressure_strategy
           @max_queue_size = max_queue_size
           @supervisor_interval = supervisor_interval
           @queue = ::MultiOpQueue::Queue.new
-          create_and_supervise_consumer!
+          @consumers = {}
+          create_and_supervise_consumers!
         end
 
         def back_pressure_strategy=(strategy)
@@ -52,36 +53,43 @@ module ActivePublisher
         def size
           # Requests might be in flight (out of the queue, but not yet published), so taking the max should be
           # good enough to make sure we're honest about the actual queue size.
-          return queue.size if consumer.nil?
-          [queue.size, consumer.sampled_queue_size].max
+          return queue.size if consumers.empty?
+          [queue.size, consumer_sampled_queue_size].max
         end
 
         private
 
-        def create_and_supervise_consumer!
-          @consumer = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
+        def create_and_supervise_consumers!
+          ::ActivePublisher.configuration.publisher_threads.times do
+            consumer_id = ::SecureRandom.uuid
+            consumers[consumer_id] = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
+            supervisor_task = ::Concurrent::TimerTask.new(:execution_interval => supervisor_interval) do
+              current_time = ::Time.now
+              consumer = consumers[consumer_id]
 
-          supervisor_task = ::Concurrent::TimerTask.new(:execution_interval => supervisor_interval) do
-            current_time = ::Time.now
+              # Consumer is lagging if it does not "tick" at least once every 10 seconds.
+              seconds_since_last_tick = current_time - consumer.last_tick_at
+              consumer_is_lagging = seconds_since_last_tick > ::ActivePublisher.configuration.max_async_publisher_lag_time
+              logger.error "ActivePublisher consumer is lagging. Last consumer tick was #{seconds_since_last_tick} seconds ago." if consumer_is_lagging
 
-            # Consumer is lagging if it does not "tick" at least once every 10 seconds.
-            seconds_since_last_tick = current_time - consumer.last_tick_at
-            consumer_is_lagging = seconds_since_last_tick > ::ActivePublisher.configuration.max_async_publisher_lag_time
-            logger.error "ActivePublisher consumer is lagging. Last consumer tick was #{seconds_since_last_tick} seconds ago." if consumer_is_lagging
+              # Check to see if we should restart the consumer.
+              if !consumer.alive? || consumer_is_lagging
+                consumer.kill rescue nil
+                consumers[consumer_id] = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
+              end
 
-            # Check to see if we should restart the consumer.
-            if !consumer.alive? || consumer_is_lagging
-              consumer.kill rescue nil
-              @consumer = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
+              # Notify the current queue size.
+              ::ActiveSupport::Notifications.instrument "async_queue_size.active_publisher", queue.size
             end
-
-            # Notify the current queue size.
-            ::ActiveSupport::Notifications.instrument "async_queue_size.active_publisher", queue.size
+            supervisor_task.execute
           end
-          supervisor_task.execute
         end
-      end
 
+        def consumer_sampled_queue_size
+          consumers.values.map(&:sampled_queue_size).max
+        end
+
+      end
     end
   end
 end

data/lib/active_publisher/async/in_memory_adapter/consumer_thread.rb

@@ -2,13 +2,7 @@ module ActivePublisher
   module Async
     module InMemoryAdapter
       class ConsumerThread
-        attr_reader :channel, :thread, :queue, :sampled_queue_size, :last_tick_at
-
-        if ::RUBY_PLATFORM == "java"
-          CHANNEL_CLOSED_ERRORS = [::MarchHare::ChannelAlreadyClosed]
-        else
-          CHANNEL_CLOSED_ERRORS = [::Bunny::ChannelAlreadyClosed]
-        end
+        attr_reader :thread, :queue, :sampled_queue_size, :last_tick_at
 
         if ::RUBY_PLATFORM == "java"
           NETWORK_ERRORS = [::MarchHare::NetworkException, ::MarchHare::ConnectionRefused,
@@ -51,32 +45,6 @@ module ActivePublisher
           end
         end
 
-        def cleanup_up_channel
-          return if channel.nil?
-          channel.close
-        rescue => error
-          ::ActivePublisher.configuration.error_handler.call(error, {:status => "Cleaning up the channel"})
-        end
-
-        def handle_current_messages_on_unknown_error(current_messages)
-          current_messages.each do |message|
-            # Degrade to single message publish ... or at least attempt to
-            begin
-              ::ActivePublisher.publish(message.route, message.payload, message.exchange_name, message.options)
-              current_messages.delete(message)
-            rescue *CHANNEL_CLOSED_ERRORS
-              # If the channel is bad, raise!
-              raise
-            rescue *PRECONDITION_ERRORS => error
-              # Delete messages if rabbitmq cannot declare the exchange (or somet other precondition failed).
-              ::ActivePublisher.configuration.error_handler.call(error, {:reason => "precondition failed", :message => message})
-              current_messages.delete(message)
-            rescue => other_error
-              ::ActivePublisher.configuration.error_handler.call(other_error, {:route => message.route, :payload => message.payload, :exchange_name => message.exchange_name, :options => message.options})
-            end
-          end
-        end
-
         def make_channel
           channel = ::ActivePublisher::Async::InMemoryAdapter::Channel.new
           channel.confirm_select if ::ActivePublisher.configuration.publisher_confirms
@@ -89,66 +57,70 @@ module ActivePublisher
 
         def start_thread
           return if alive?
-          @thread = ::Thread.new { start_consuming_thread }
-        end
-
-        def start_consuming_thread
-          loop do
-            # Sample the queue size so we don't shutdown when messages are in flight.
-            @sampled_queue_size = queue.size
-            current_messages = queue.pop_up_to(50, :timeout => 0.1)
-            update_last_tick_at
-            # If the queue is empty, we should continue to update to "last_tick_at" time.
-            next if current_messages.nil?
-
-            @channel ||= make_channel
-
-            # We only look at active publisher messages. Everything else is dropped.
-            current_messages.select! { |message| message.is_a?(::ActivePublisher::Message) }
-
-            begin
-              # Only open a single connection for each group of messages to an exchange
-              current_messages.group_by(&:exchange_name).each do |exchange_name, messages|
-                publish_all(exchange_name, messages)
-                current_messages -= messages
+          @thread = ::Thread.new do
+            loop do
+              # Sample the queue size so we don't shutdown when messages are in flight.
+              @sampled_queue_size = queue.size
+              current_messages = queue.pop_up_to(50, :timeout => 0.1)
+              update_last_tick_at
+              # If the queue is empty, we should continue to update to "last_tick_at" time.
+              next if current_messages.nil?
+
+              # We only look at active publisher messages. Everything else is dropped.
+              current_messages.select! { |message| message.is_a?(::ActivePublisher::Message) }
+
+              begin
+                @channel ||= make_channel
+
+                # Only open a single connection for each group of messages to an exchange
+                current_messages.group_by(&:exchange_name).each do |exchange_name, messages|
+                  publish_all(@channel, exchange_name, messages)
+                  current_messages -= messages
+                end
+              rescue *NETWORK_ERRORS
+                # Sleep because connection is down
+                await_network_reconnect
+              rescue => unknown_error
+                ::ActivePublisher.configuration.error_handler.call(unknown_error, {:number_of_messages => current_messages.size})
+                current_messages.each do |message|
+                  # Degrade to single message publish ... or at least attempt to
+                  begin
+                    ::ActivePublisher.publish(message.route, message.payload, message.exchange_name, message.options)
+                    current_messages.delete(message)
+                  rescue *PRECONDITION_ERRORS => error
+                    # Delete messages if rabbitmq cannot declare the exchange (or somet other precondition failed).
+                    ::ActivePublisher.configuration.error_handler.call(error, {:reason => "precondition failed", :message => message})
+                    current_messages.delete(message)
+                  rescue => individual_error
+                    ::ActivePublisher.configuration.error_handler.call(individual_error, {:route => message.route, :payload => message.payload, :exchange_name => message.exchange_name, :options => message.options})
+                  end
+                end
+
+                # TODO: Find a way to bubble this out of the thread for logging purposes.
+                # Reraise the error out of the publisher loop. The Supervisor will restart the consumer.
+                raise unknown_error
+              ensure
+                # Always requeue anything that gets stuck.
+                queue.concat(current_messages) if current_messages && !current_messages.empty?
               end
-            rescue *CHANNEL_CLOSED_ERRORS
-              # If the channel is bad, raise without sending one-by-one!
-              raise
-            rescue *NETWORK_ERRORS
-              # Sleep because connection is down
-              await_network_reconnect
-            rescue => unknown_error
-              ::ActivePublisher.configuration.error_handler.call(unknown_error, {:number_of_messages => current_messages.size})
-
-              # Attempt to deliver a message one-by-one. Raise if a closed channel error appears.
-              handle_current_messages_on_unknown_error(current_messages)
-
-              # TODO: Find a way to bubble this out of the thread for logging purposes.
-              # Reraise the error out of the publisher loop. The Supervisor will restart the consumer.
-              raise unknown_error
-            ensure
-              # Always requeue anything that gets stuck.
-              queue.concat(current_messages) if current_messages && !current_messages.empty?
             end
           end
-        ensure
-          cleanup_up_channel
         end
 
-        def publish_all(exchange_name, messages)
-          exchange = channel.topic(exchange_name)
-          messages.each do |message|
-            fail ::ActivePublisher::ExchangeMismatchError, "bulk publish messages must match publish_all exchange_name" if message.exchange_name != exchange_name
-            ::ActiveSupport::Notifications.instrument "message_published.active_publisher", :route => message.route, :message_count => 1 do
+        def publish_all(channel, exchange_name, messages)
+          ::ActiveSupport::Notifications.instrument "message_published.active_publisher", :message_count => messages.size do
+            exchange = channel.topic(exchange_name)
+            messages.each do |message|
+              fail ActivePublisher::ExchangeMismatchError, "bulk publish messages must match publish_all exchange_name" if message.exchange_name != exchange_name
+
               options = ::ActivePublisher.publishing_options(message.route, message.options || {})
               exchange.publish(message.payload, options)
             end
+            wait_for_confirms(channel)
           end
-          wait_for_confirms
         end
 
-        def wait_for_confirms
+        def wait_for_confirms(channel)
           return true unless channel.using_publisher_confirms?
           channel.wait_for_confirms(::ActivePublisher.configuration.publisher_confirms_timeout)
         end

data/lib/active_publisher/configuration.rb

@@ -11,6 +11,7 @@ module ActivePublisher
                   :network_recovery_interval,
                   :password,
                   :port,
+                  :publisher_threads,
                   :publisher_confirms,
                   :publisher_confirms_timeout,
                   :seconds_to_wait_for_graceful_shutdown,
@@ -39,6 +40,7 @@ module ActivePublisher
       :max_async_publisher_lag_time => 10,
       :network_recovery_interval => NETWORK_RECOVERY_INTERVAL,
       :port => 5672,
+      :publisher_threads => 1,
       :publisher_confirms => false,
       :publisher_confirms_timeout => 5_000, #specified as a number of milliseconds
       :seconds_to_wait_for_graceful_shutdown => 30,
@@ -63,8 +65,8 @@ module ActivePublisher
 
           yaml_config = attempt_to_load_yaml_file(env)
           DEFAULTS.each_pair do |key, value|
-            exists, setting = fetch_config_value(key, cli_options, yaml_config)
-            ::ActivePublisher.configuration.public_send("#{key}=", setting) if exists
+            setting = cli_options[key] || cli_options[key.to_s] || yaml_config[key] || yaml_config[key.to_s]
+            ::ActivePublisher.configuration.public_send("#{key}=", setting) if setting
           end
 
           true
@@ -88,15 +90,6 @@ module ActivePublisher
     end
     private_class_method :attempt_to_load_yaml_file
 
-    def self.fetch_config_value(key, cli_options, yaml_config)
-      return [true, cli_options[key]] if cli_options.key?(key)
-      return [true, cli_options[key.to_s]] if cli_options.key?(key.to_s)
-      return [true, yaml_config[key]] if yaml_config.key?(key)
-      return [true, yaml_config[key.to_s]] if yaml_config.key?(key.to_s)
-      [false, nil]
-    end
-    private_class_method :fetch_config_value
-
     ##
     # Instance Methods
     #

data/lib/active_publisher/connection.rb

@@ -31,22 +31,9 @@ module ActivePublisher
     def self.create_connection
       if ::RUBY_PLATFORM == "java"
         connection = ::MarchHare.connect(connection_options)
-        connection.on_blocked do |reason|
-          on_blocked(reason)
-        end
-        connection.on_unblocked do
-          on_unblocked
-        end
-        connection
       else
         connection = ::Bunny.new(connection_options)
         connection.start
-        connection.on_blocked do |blocked_message|
-          on_blocked(blocked_message.reason)
-        end
-        connection.on_unblocked do
-          on_unblocked
-        end
         connection
       end
     end
@@ -71,15 +58,5 @@ module ActivePublisher
       }
     end
     private_class_method :connection_options
-
-    def self.on_blocked(reason)
-      ::ActiveSupport::Notifications.instrument("connection_blocked.active_publisher", :reason => reason)
-    end
-    private_class_method :on_blocked
-
-    def self.on_unblocked
-      ::ActiveSupport::Notifications.instrument("connection_unblocked.active_publisher")
-    end
-    private_class_method :on_unblocked
   end
 end

data/lib/active_publisher/version.rb

@@ -1,3 +1,3 @@
 module ActivePublisher
-  VERSION = "1.2.5"
+  VERSION = "1.3.0.pre0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: active_publisher
 version: !ruby/object:Gem::Version
-  version: 1.2.5
+  version: 1.3.0.pre0
 platform: java
 authors:
 - Brian Stien
@@ -12,7 +12,7 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-11-23 00:00:00.000000000 Z
+date: 2018-03-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -143,17 +143,17 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '10.0'
   name: rake
   prerelease: false
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '10.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -179,9 +179,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".circleci/config.yml"
 - ".gitignore"
 - ".rspec"
+- ".travis.yml"
 - CODE_OF_CONDUCT.md
 - Gemfile
 - LICENSE.txt
@@ -219,12 +219,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project:
-rubygems_version: 2.7.9
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
 summary: Aims to make publishing work across MRI and jRuby painless and add some nice

data/.circleci/config.yml

@@ -1,56 +0,0 @@
-# Inspired by: http://mikebian.co/running-tests-against-multiple-ruby-versions-using-circleci/
-
-version: 2.1
-
-orbs:
-  ruby: circleci/ruby@1.1
-
-jobs:
-  test:
-    parallelism: 1
-    parameters:
-      ruby-image:
-        type: string
-    docker:
-      - image: << parameters.ruby-image >>
-      - image: rabbitmq
-
-    steps:
-      - checkout
-      - run:
-          name: install dockerize
-          command: wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz
-          environment:
-            DOCKERIZE_VERSION: v0.3.0
-      - run:
-          name: Wait for rabbitmq
-          command: dockerize -wait tcp://localhost:5672 -timeout 1m
-      - run:
-          name: Install bundler
-          command: gem install bundler
-      - run:
-          name: Which bundler?
-          command: bundle -v
-      - run:
-          name: bundle install
-          command: bundle install
-      - run:
-          name: rspec
-          command: bundle exec rspec
-
-# strangely, there seems to be very little documentation about exactly how martix builds work.
-# By defining a param inside your job definition, Circle CI will automatically spawn a job for
-# unique param value passed via `matrix`. Neat!
-# https://circleci.com/blog/circleci-matrix-jobs/
-workflows:
-  build_and_test:
-    jobs:
-      - test:
-          matrix:
-            parameters:
-              ruby-image:
-                - circleci/ruby:2.5
-                - circleci/ruby:2.6
-                - circleci/ruby:2.7
-                - circleci/jruby:9.1
-                - circleci/jruby:9.2