active_publisher 1.3.0.pre0 → 1.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 45653514bae6a0a4516505d102cbbdf19efec6a0
-  data.tar.gz: 2c3ca2a6b7fb9f38f56885ccb59dfdc327a9df9a
+SHA256:
+  metadata.gz: 1f52f284362383d67ea86b1fd21a9bfa855e32bea6ad77c7c482fc47ece6ef09
+  data.tar.gz: 4b09f1fd7b793fbca7012a4f6d01c9797029529a69c7e8482c18c3af3cfeb51f
 SHA512:
-  metadata.gz: 21da8df20e862db89aa28e2d5161b29e14f1ada5f2f46e1f7c9e67cf7fc1a0d27366b9b091242c85f1ecf53a08a571069d785c21d602e9e2c80ce66ab436bc50
-  data.tar.gz: 4c35c461363239c66846880650f951dcad7d20976c3473513eb87d0ed4f5ac716d14b1fe8f64bb2133bd12c6281f0222b7dffd61eab1b576379be4bc40048c60
+  metadata.gz: 3ab3dfaa3727a5e56c639aacdc12158c30fad925ea112701f48c7c9d4dcfab0028fb5c799841e8d9920ba78d83e42edb0b89fc22f4cc3f6fcdb77093937c77b0
+  data.tar.gz: d4a514867fea7d76fae78aff4b083bd1044aee9e775f4e57d4d72ea12429201b0089033eca3b8cf7d6544d0c5c5b027735f6e178580575f26d75bd1426429ea2

data/.circleci/config.yml

@@ -0,0 +1,56 @@
+# Inspired by: http://mikebian.co/running-tests-against-multiple-ruby-versions-using-circleci/
+
+version: 2.1
+
+orbs:
+  ruby: circleci/ruby@1.1
+
+jobs:
+  test:
+    parallelism: 1
+    parameters:
+      ruby-image:
+        type: string
+    docker:
+      - image: << parameters.ruby-image >>
+      - image: rabbitmq
+
+    steps:
+      - checkout
+      - run:
+          name: install dockerize
+          command: wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz
+          environment:
+            DOCKERIZE_VERSION: v0.3.0
+      - run:
+          name: Wait for rabbitmq
+          command: dockerize -wait tcp://localhost:5672 -timeout 1m
+      - run:
+          name: Install bundler
+          command: gem install bundler
+      - run:
+          name: Which bundler?
+          command: bundle -v
+      - run:
+          name: bundle install
+          command: bundle install
+      - run:
+          name: rspec
+          command: bundle exec rspec
+
+# strangely, there seems to be very little documentation about exactly how martix builds work.
+# By defining a param inside your job definition, Circle CI will automatically spawn a job for
+# unique param value passed via `matrix`. Neat!
+# https://circleci.com/blog/circleci-matrix-jobs/
+workflows:
+  build_and_test:
+    jobs:
+      - test:
+          matrix:
+            parameters:
+              ruby-image:
+                - circleci/ruby:2.5
+                - circleci/ruby:2.6
+                - circleci/ruby:2.7
+                - circleci/jruby:9.1
+                - circleci/jruby:9.2

data/active_publisher.gemspec

@@ -35,6 +35,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "connection_pool"
   spec.add_development_dependency "fakeredis"
   spec.add_development_dependency "pry"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec", "~> 3.2"
 end

data/lib/active_publisher/async/in_memory_adapter/async_queue.rb

@@ -76,6 +76,7 @@ module ActivePublisher
               if !consumer.alive? || consumer_is_lagging
                 consumer.kill rescue nil
                 consumers[consumer_id] = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
+                ::ActiveSupport::Notifications.instrument "async_queue.thread_restart"
               end
 
               # Notify the current queue size.

data/lib/active_publisher/async/in_memory_adapter/consumer_thread.rb

@@ -2,7 +2,13 @@ module ActivePublisher
   module Async
     module InMemoryAdapter
       class ConsumerThread
-        attr_reader :thread, :queue, :sampled_queue_size, :last_tick_at
+        attr_reader :channel, :flush_max, :thread, :queue, :sampled_queue_size, :last_tick_at
+
+        if ::RUBY_PLATFORM == "java"
+          CHANNEL_CLOSED_ERRORS = [::MarchHare::ChannelAlreadyClosed]
+        else
+          CHANNEL_CLOSED_ERRORS = [::Bunny::ChannelAlreadyClosed]
+        end
 
         if ::RUBY_PLATFORM == "java"
           NETWORK_ERRORS = [::MarchHare::NetworkException, ::MarchHare::ConnectionRefused,
@@ -21,6 +27,7 @@ module ActivePublisher
         def initialize(listen_queue)
           @queue = listen_queue
           @sampled_queue_size = queue.size
+          @flush_max = ::ActivePublisher.configuration.messages_per_batch
 
           update_last_tick_at
           start_thread
@@ -45,6 +52,32 @@ module ActivePublisher
           end
         end
 
+        def cleanup_up_channel
+          return if channel.nil?
+          channel.close
+        rescue => error
+          ::ActivePublisher.configuration.error_handler.call(error, {:status => "Cleaning up the channel"})
+        end
+
+        def handle_current_messages_on_unknown_error(current_messages)
+          current_messages.each do |message|
+            # Degrade to single message publish ... or at least attempt to
+            begin
+              ::ActivePublisher.publish(message.route, message.payload, message.exchange_name, message.options)
+              current_messages.delete(message)
+            rescue *CHANNEL_CLOSED_ERRORS
+              # If the channel is bad, raise!
+              raise
+            rescue *PRECONDITION_ERRORS => error
+              # Delete messages if rabbitmq cannot declare the exchange (or somet other precondition failed).
+              ::ActivePublisher.configuration.error_handler.call(error, {:reason => "precondition failed", :message => message})
+              current_messages.delete(message)
+            rescue => other_error
+              ::ActivePublisher.configuration.error_handler.call(other_error, {:route => message.route, :payload => message.payload, :exchange_name => message.exchange_name, :options => message.options})
+            end
+          end
+        end
+
         def make_channel
           channel = ::ActivePublisher::Async::InMemoryAdapter::Channel.new
           channel.confirm_select if ::ActivePublisher.configuration.publisher_confirms
@@ -57,72 +90,70 @@ module ActivePublisher
 
         def start_thread
           return if alive?
-          @thread = ::Thread.new do
-            loop do
-              # Sample the queue size so we don't shutdown when messages are in flight.
-              @sampled_queue_size = queue.size
-              current_messages = queue.pop_up_to(50, :timeout => 0.1)
-              update_last_tick_at
-              # If the queue is empty, we should continue to update to "last_tick_at" time.
-              next if current_messages.nil?
-
-              # We only look at active publisher messages. Everything else is dropped.
-              current_messages.select! { |message| message.is_a?(::ActivePublisher::Message) }
-
-              begin
-                @channel ||= make_channel
-
-                # Only open a single connection for each group of messages to an exchange
-                current_messages.group_by(&:exchange_name).each do |exchange_name, messages|
-                  publish_all(@channel, exchange_name, messages)
-                  current_messages -= messages
-                end
-              rescue *NETWORK_ERRORS
-                # Sleep because connection is down
-                await_network_reconnect
-              rescue => unknown_error
-                ::ActivePublisher.configuration.error_handler.call(unknown_error, {:number_of_messages => current_messages.size})
-                current_messages.each do |message|
-                  # Degrade to single message publish ... or at least attempt to
-                  begin
-                    ::ActivePublisher.publish(message.route, message.payload, message.exchange_name, message.options)
-                    current_messages.delete(message)
-                  rescue *PRECONDITION_ERRORS => error
-                    # Delete messages if rabbitmq cannot declare the exchange (or somet other precondition failed).
-                    ::ActivePublisher.configuration.error_handler.call(error, {:reason => "precondition failed", :message => message})
-                    current_messages.delete(message)
-                  rescue => individual_error
-                    ::ActivePublisher.configuration.error_handler.call(individual_error, {:route => message.route, :payload => message.payload, :exchange_name => message.exchange_name, :options => message.options})
-                  end
-                end
-
-                # TODO: Find a way to bubble this out of the thread for logging purposes.
-                # Reraise the error out of the publisher loop. The Supervisor will restart the consumer.
-                raise unknown_error
-              ensure
-                # Always requeue anything that gets stuck.
-                queue.concat(current_messages) if current_messages && !current_messages.empty?
+          @thread = ::Thread.new { start_consuming_thread }
+        end
+
+        def start_consuming_thread
+          loop do
+            # Sample the queue size so we don't shutdown when messages are in flight.
+            @sampled_queue_size = queue.size
+            current_messages = queue.pop_up_to(flush_max, :timeout => 0.1)
+            update_last_tick_at
+            # If the queue is empty, we should continue to update to "last_tick_at" time.
+            next if current_messages.nil?
+
+            @channel ||= make_channel
+
+            # We only look at active publisher messages. Everything else is dropped.
+            current_messages.select! { |message| message.is_a?(::ActivePublisher::Message) }
+
+            begin
+              # Only open a single connection for each group of messages to an exchange
+              current_messages.group_by(&:exchange_name).each do |exchange_name, messages|
+                publish_all(exchange_name, messages)
+                current_messages -= messages
               end
+            rescue *CHANNEL_CLOSED_ERRORS
+              # If the channel is bad, raise without sending one-by-one!
+              raise
+            rescue *NETWORK_ERRORS
+              # Sleep because connection is down
+              await_network_reconnect
+            rescue => unknown_error
+              ::ActivePublisher.configuration.error_handler.call(unknown_error, {:number_of_messages => current_messages.size})
+
+              # Attempt to deliver a message one-by-one. Raise if a closed channel error appears.
+              handle_current_messages_on_unknown_error(current_messages)
+
+              # TODO: Find a way to bubble this out of the thread for logging purposes.
+              # Reraise the error out of the publisher loop. The Supervisor will restart the consumer.
+              raise unknown_error
+            ensure
+              # Always requeue anything that gets stuck.
+              queue.concat(current_messages) if current_messages && !current_messages.empty?
             end
           end
+        ensure
+          cleanup_up_channel
         end
 
-        def publish_all(channel, exchange_name, messages)
-          ::ActiveSupport::Notifications.instrument "message_published.active_publisher", :message_count => messages.size do
-            exchange = channel.topic(exchange_name)
-            messages.each do |message|
-              fail ActivePublisher::ExchangeMismatchError, "bulk publish messages must match publish_all exchange_name" if message.exchange_name != exchange_name
-
+        def publish_all(exchange_name, messages)
+          exchange = channel.topic(exchange_name)
+          messages.each do |message|
+            fail ::ActivePublisher::ExchangeMismatchError, "bulk publish messages must match publish_all exchange_name" if message.exchange_name != exchange_name
+            ::ActiveSupport::Notifications.instrument "message_published.active_publisher", :route => message.route, :message_count => 1 do
               options = ::ActivePublisher.publishing_options(message.route, message.options || {})
               exchange.publish(message.payload, options)
             end
-            wait_for_confirms(channel)
           end
+          wait_for_confirms
         end
 
-        def wait_for_confirms(channel)
+        def wait_for_confirms
           return true unless channel.using_publisher_confirms?
-          channel.wait_for_confirms(::ActivePublisher.configuration.publisher_confirms_timeout)
+          ::ActiveSupport::Notifications.instrument "publishes_confirmed.active_publisher" do
+            channel.wait_for_confirms(::ActivePublisher.configuration.publisher_confirms_timeout)
+          end
         end
       end
     end

data/lib/active_publisher/async/redis_adapter/consumer.rb

@@ -9,28 +9,33 @@ module ActivePublisher
           :timeout_interval => 5, # seconds
         }
 
-        attr_reader :consumer, :queue, :supervisor
+        attr_reader :consumers, :queue, :supervisor
 
         def initialize(redis_pool)
           @queue = ::ActivePublisher::Async::RedisAdapter::RedisMultiPopQueue.new(redis_pool, ::ActivePublisher::Async::RedisAdapter::REDIS_LIST_KEY)
-          create_and_supervise_consumer!
+          @consumers = {}
+          create_and_supervise_consumers!
         end
 
-        def create_and_supervise_consumer!
-          @consumer = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
-
-          supervisor_task = ::Concurrent::TimerTask.new(SUPERVISOR_INTERVAL) do
-            # This may also be the place to start additional publishers when we are getting backed up ... ?
-            unless consumer.alive?
-              consumer.kill rescue nil
-              @consumer = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
+        def create_and_supervise_consumers!
+          ::ActivePublisher.configuration.publisher_threads.times do
+            consumer_id = ::SecureRandom.uuid
+            consumers[consumer_id] = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
+
+            supervisor_task = ::Concurrent::TimerTask.new(SUPERVISOR_INTERVAL) do
+              consumer = consumers[consumer_id]
+              unless consumer.alive?
+                consumer.kill rescue nil
+                consumers[consumer_id] = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
+                ::ActiveSupport::Notifications.instrument "async_queue.thread_restart"
+              end
+
+              # Notify the current queue size.
+              ::ActiveSupport::Notifications.instrument "redis_async_queue_size.active_publisher", queue.size
             end
 
-            # Notify the current queue size.
-            ::ActiveSupport::Notifications.instrument "redis_async_queue_size.active_publisher", queue.size
+            supervisor_task.execute
           end
-
-          supervisor_task.execute
         end
 
         def size

data/lib/active_publisher/async/redis_adapter/redis_multi_pop_queue.rb

@@ -77,9 +77,9 @@ module ActivePublisher
           messages = []
           multi_response = []
           redis_pool.with do |redis|
-            multi_response = redis.multi do
-              redis.lrange(list_key, 0, number - 1)
-              redis.ltrim(list_key, number, -1)
+            multi_response = redis.multi do |pipeline|
+              pipeline.lrange(list_key, 0, number - 1)
+              pipeline.ltrim(list_key, number, -1)
             end
           end
 

data/lib/active_publisher/async/redis_adapter.rb

@@ -19,7 +19,7 @@ module ActivePublisher
         }
         include ::ActivePublisher::Logging
 
-        attr_reader :async_queue, :redis_pool, :queue
+        attr_reader :async_queue, :flush_max, :flush_min, :redis_pool, :queue
 
         def initialize(new_redis_pool)
           logger.info "Starting redis publisher adapter"
@@ -27,6 +27,8 @@ module ActivePublisher
           @redis_pool = new_redis_pool
           @async_queue = ::ActivePublisher::Async::RedisAdapter::Consumer.new(redis_pool)
           @queue = ::MultiOpQueue::Queue.new
+          @flush_max = ::ActivePublisher.configuration.messages_per_batch
+          @flush_min = @flush_max / 2
 
           supervisor_task = ::Concurrent::TimerTask.new(SUPERVISOR_INTERVAL) do
             queue_size = queue.size
@@ -41,7 +43,7 @@ module ActivePublisher
         def publish(route, payload, exchange_name, options = {})
           message = ::ActivePublisher::Message.new(route, payload, exchange_name, options)
           queue << ::Marshal.dump(message)
-          flush_queue! if queue.size >= 20 || options[:flush_queue]
+          flush_queue! if queue.size >= flush_min || options[:flush_queue]
 
           nil
         end
@@ -58,7 +60,7 @@ module ActivePublisher
 
         def flush_queue!
           return if queue.empty?
-          encoded_messages = queue.pop_up_to(25, :timeout => 0.001)
+          encoded_messages = queue.pop_up_to(flush_max, :timeout => 0.001)
 
           return if encoded_messages.nil?
           return unless encoded_messages.respond_to?(:each)

data/lib/active_publisher/configuration.rb

@@ -8,6 +8,7 @@ module ActivePublisher
                   :host,
                   :hosts,
                   :max_async_publisher_lag_time,
+                  :messages_per_batch,
                   :network_recovery_interval,
                   :password,
                   :port,
@@ -37,6 +38,7 @@ module ActivePublisher
       :host => "localhost",
       :hosts => [],
       :password => "guest",
+      :messages_per_batch => 25,
       :max_async_publisher_lag_time => 10,
       :network_recovery_interval => NETWORK_RECOVERY_INTERVAL,
       :port => 5672,
@@ -65,8 +67,8 @@ module ActivePublisher
 
           yaml_config = attempt_to_load_yaml_file(env)
           DEFAULTS.each_pair do |key, value|
-            setting = cli_options[key] || cli_options[key.to_s] || yaml_config[key] || yaml_config[key.to_s]
-            ::ActivePublisher.configuration.public_send("#{key}=", setting) if setting
+            exists, setting = fetch_config_value(key, cli_options, yaml_config)
+            ::ActivePublisher.configuration.public_send("#{key}=", setting) if exists
           end
 
           true
@@ -90,6 +92,15 @@ module ActivePublisher
     end
     private_class_method :attempt_to_load_yaml_file
 
+    def self.fetch_config_value(key, cli_options, yaml_config)
+      return [true, cli_options[key]] if cli_options.key?(key)
+      return [true, cli_options[key.to_s]] if cli_options.key?(key.to_s)
+      return [true, yaml_config[key]] if yaml_config.key?(key)
+      return [true, yaml_config[key.to_s]] if yaml_config.key?(key.to_s)
+      [false, nil]
+    end
+    private_class_method :fetch_config_value
+
     ##
     # Instance Methods
     #

data/lib/active_publisher/connection.rb

@@ -31,9 +31,22 @@ module ActivePublisher
     def self.create_connection
       if ::RUBY_PLATFORM == "java"
         connection = ::MarchHare.connect(connection_options)
+        connection.on_blocked do |reason|
+          on_blocked(reason)
+        end
+        connection.on_unblocked do
+          on_unblocked
+        end
+        connection
       else
         connection = ::Bunny.new(connection_options)
         connection.start
+        connection.on_blocked do |blocked_message|
+          on_blocked(blocked_message.reason)
+        end
+        connection.on_unblocked do
+          on_unblocked
+        end
         connection
       end
     end
@@ -58,5 +71,15 @@ module ActivePublisher
       }
     end
     private_class_method :connection_options
+
+    def self.on_blocked(reason)
+      ::ActiveSupport::Notifications.instrument("connection_blocked.active_publisher", :reason => reason)
+    end
+    private_class_method :on_blocked
+
+    def self.on_unblocked
+      ::ActiveSupport::Notifications.instrument("connection_unblocked.active_publisher")
+    end
+    private_class_method :on_unblocked
   end
 end

data/lib/active_publisher/version.rb

@@ -1,3 +1,3 @@
 module ActivePublisher
-  VERSION = "1.3.0.pre0"
+  VERSION = "1.3.2"
 end

data/lib/active_publisher.rb

@@ -36,7 +36,7 @@ module ActivePublisher
   # @param [Hash] options hash to set message parameters (e.g. headers)
   def self.publish(route, payload, exchange_name, options = {})
     with_exchange(exchange_name) do |exchange|
-      ::ActiveSupport::Notifications.instrument "message_published.active_publisher" do
+      ::ActiveSupport::Notifications.instrument "message_published.active_publisher", :route => route do
         exchange.publish(payload, publishing_options(route, options))
       end
     end
@@ -52,7 +52,7 @@ module ActivePublisher
         fail ActivePublisher::ExchangeMismatchError, "bulk publish messages must match publish_all exchange_name" if message.exchange_name != exchange_name
 
         begin
-          ::ActiveSupport::Notifications.instrument "message_published.active_publisher" do
+          ::ActiveSupport::Notifications.instrument "message_published.active_publisher", :route => message.route do
             exchange.publish(message.payload, publishing_options(message.route, message.options || {}))
           end
         rescue

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: active_publisher
 version: !ruby/object:Gem::Version
-  version: 1.3.0.pre0
+  version: 1.3.2
 platform: ruby
 authors:
 - Brian Stien
@@ -9,10 +9,10 @@ authors:
 - Brandon Dewitt
 - Devin Christensen
 - Michael Ries
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2018-03-13 00:00:00.000000000 Z
+date: 2022-02-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -144,16 +144,16 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -179,9 +179,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".circleci/config.yml"
 - ".gitignore"
 - ".rspec"
-- ".travis.yml"
 - CODE_OF_CONDUCT.md
 - Gemfile
 - LICENSE.txt
@@ -208,7 +208,7 @@ homepage: https://github.com/mxenabled/active_publisher
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -219,13 +219,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.13
-signing_key: 
+rubygems_version: 3.0.1
+signing_key:
 specification_version: 4
 summary: Aims to make publishing work across MRI and jRuby painless and add some nice
   features like automatially publishing lifecycle events for ActiveRecord models.

data/.travis.yml

@@ -1,9 +0,0 @@
-language: ruby
-rvm:
- - 2.3
- - 2.4
- - jruby-9.1.12.0
-services:
- - rabbitmq
-sudo: false
-cache: bundler