active_publisher 0.1.5-java → 0.2.0.pre-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 658bc2636e515d48b346bde7f92421e7c9dfe9dd
-  data.tar.gz: 8b454665dbc31565a67184d4b699463f685049be
+  metadata.gz: 63df17dc2a58125dcc18b1cb9c7d20f0517da0ac
+  data.tar.gz: 9bd8dcd05da48bde7c14ff49a3dc10928ab609a6
 SHA512:
-  metadata.gz: 3e3ef0abdc6abc0c4bc3995966c8197f5c803581ea549c109762dff9b40b7fc59ce39c12b00079d0b596e5b6b90ab298cfbc53a6b116906409d33940cb1e6e72
-  data.tar.gz: 096f63f02a40ace574301c542d657a85f0a23c151ec058ae9ca9f3179b845b01e0d350aae53d41705225f4c06307f74b114e298da552e18433a2c7480a3d8a3e
+  metadata.gz: e8dbdc550e66d0d4182ed96333a03e37ab89466371bc3bc4634ff6d78e1f02e8daeafb5ed3b13f0bc3cb91d793af46ee6954ba2519e148122d8b2dde569a025c
+  data.tar.gz: d6a7965709096d9f9828afe0655982aaa5ddafa87b3256e44d6c815df4b6467af64bd0236d5f3c49edb9dd366f28dbb2e3011c1373ebd86a8ea9928bf8418bb6

data/.gitignore

@@ -1,3 +1,4 @@
+.ruby-*
 /.bundle/
 /.yardoc
 /Gemfile.lock

data/README.md

@@ -20,9 +20,76 @@ Or install it yourself as:
 
     $ gem install active_publisher
 
+## Configuration
+
+ActivePublisher will use a `config/active_publisher.yml` or `config/action_subscriber.yml` automatically.
+
+Create a `config/active_publisher.yml` similar to a database.yml, with your configuration nested in your environments keys.
+
+```yaml
+default: &default
+  host: localhost
+  username: guest
+  password: guest
+
+development:
+  <<: *default
+
+test:
+  <<: *default
+
+production:
+  <<: *default
+  host: <%= ENV['RABBIT_MQ_HOST'] %>
+  username: <%= ENV['RABBIT_MQ_USERNAME'] %>
+  password: <%= ENV['RABBIT_MQ_PASSWORD'] %>
+```
+
+Defaults for the configuration are:
+```ruby
+{
+  :heartbeat => 5,
+  :host => "localhost",
+  :hosts => [],
+  :port => 5672,
+  :publisher_confirms => false,
+  :seconds_to_wait_for_graceful_shutdown => 30,
+  :timeout => 1,
+  :username => "guest",
+  :password => "guest",
+  :virtual_host => "/"
+}
+```
+
 ## Usage
 
-TODO: Write usage instructions here
+Basic publishing is simple.
+
+```ruby
+  # @param [String] route The routing key to use for this message.
+  # @param [String] payload The message you are sending. Should already be encoded as a string.
+  # @param [String] exchange The exchange you want to publish to.
+  # @param [Hash] options hash to set message parameters (e.g. headers)
+
+  ::ActivePublisher.publish("user.created", user.to_json, "events", {})
+```
+
+
+Async publishing is as simple as configuring the async publishing adapter and running `publish_sync` the same was as publish.
+You can use the `::ActivePublisher::Async::InMemoryAdapter` that ships with `ActivePublisher`.
+
+
+`initializers/active_publisher.rb`
+```ruby
+require "active_publisher"
+::ActivePublisher::Async.publisher_adapter = ::ActivePublisher::Async::InMemoryAdapter.new
+
+```
+
+```ruby
+::ActivePublisher.publish_async("user.created", user.to_json, "events", {})
+```
+
 
 ## Development
 

data/active_publisher.gemspec

@@ -26,6 +26,8 @@ Gem::Specification.new do |spec|
     spec.add_dependency 'bunny', '~> 2.1'
   end
 
+  spec.add_dependency 'multi_op_queue', '>= 0.1.2'
+
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "pry"
   spec.add_development_dependency "rake", "~> 10.0"

data/lib/active_publisher/async/in_memory_adapter/async_queue.rb

@@ -0,0 +1,61 @@
+module ActivePublisher
+  module Async
+    module InMemoryAdapter
+      class AsyncQueue
+        include ::ActivePublisher::Logging
+
+        attr_accessor :drop_messages_when_queue_full,
+          :max_queue_size,
+          :supervisor_interval
+
+        attr_reader :consumer, :queue, :supervisor
+
+        def initialize(drop_messages_when_queue_full, max_queue_size, supervisor_interval)
+          @drop_messages_when_queue_full = drop_messages_when_queue_full
+          @max_queue_size = max_queue_size
+          @supervisor_interval = supervisor_interval
+          @queue = ::MultiOpQueue::Queue.new
+          create_and_supervise_consumer!
+        end
+
+        def push(message)
+          # default of 1_000_000 messages
+          if queue.size > max_queue_size
+            # Drop messages if the queue is full and we were configured to do so
+            return if drop_messages_when_queue_full
+
+            # By default we will raise an error to push the responsibility onto the caller
+            fail ::ActivePublisher::Async::InMemoryAdapter::UnableToPersistMessageError, "Queue is full, messages will be dropped."
+          end
+
+          queue.push(message)
+        end
+
+        def size
+          queue.size
+        end
+
+        private
+
+        def create_and_supervise_consumer!
+          @consumer = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
+          @supervisor = ::Thread.new do
+            loop do
+              unless consumer.alive?
+                # We might need to requeue the last messages popped
+                current_consumer_messages = consumer.current_messages
+                queue.concat(current_consumer_messages) unless current_consumer_messages.empty?
+                consumer.kill
+                @consumer = ::ActivePublisher::Async::InMemoryAdapter::ConsumerThread.new(queue)
+              end
+
+              # Pause before checking the consumer again.
+              sleep supervisor_interval
+            end
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/active_publisher/async/in_memory_adapter/consumer_thread.rb

@@ -0,0 +1,89 @@
+module ActivePublisher
+  module Async
+    module InMemoryAdapter
+      class ConsumerThread
+        attr_reader :current_messages, :thread, :queue
+
+        if ::RUBY_PLATFORM == "java"
+          NETWORK_ERRORS = [::MarchHare::Exception, ::Java::ComRabbitmqClient::AlreadyClosedException, ::Java::JavaIo::IOException].freeze
+        else
+          NETWORK_ERRORS = [::Bunny::Exception, ::Timeout::Error, ::IOError].freeze
+        end
+
+        def initialize(listen_queue)
+          @current_messages = []
+          @queue = listen_queue
+          start_thread
+        end
+
+        def alive?
+          @thread && @thread.alive?
+        end
+
+        def kill
+          @thread.kill if @thread
+          @thread = nil
+        end
+
+        private
+
+        def await_network_reconnect
+          if defined?(ActivePublisher::RabbitConnection)
+            sleep ::ActivePublisher::RabbitConnection::NETWORK_RECOVERY_INTERVAL
+          else
+            sleep 0.1
+          end
+        end
+
+        def start_thread
+          return if alive?
+          @thread = ::Thread.new do
+            loop do
+              # Write "current_messages" so we can requeue should something happen to the consumer.
+              @current_messages.concat(queue.pop_up_to(20))
+
+              begin
+                # Only open a single connection for each group of messages to an exchange
+                messages_to_retry = []
+                @current_messages.group_by(&:exchange_name).each do |exchange_name, messages|
+                  begin
+                    ::ActivePublisher.publish_all(exchange_name, messages)
+                  ensure
+                    messages_to_retry.concat(messages)
+                  end
+                end
+
+                # Reset
+                @current_messages = []
+              rescue *NETWORK_ERRORS
+                # Sleep because connection is down
+                await_network_reconnect
+                @current_messages.concat(messages_to_retry)
+
+                # Requeue and try again.
+                queue.concat(@current_messages) unless @current_messages.empty?
+              rescue => unknown_error
+                @current_messages.concat(messages_to_retry)
+                @current_messages.each do |message|
+                  # Degrade to single message publish ... or at least attempt to
+                  begin
+                    ::ActivePublisher.publish(message.route, message.payload, message.exchange_name, message.options)
+                  rescue => error
+                    ::ActivePublisher.configuration.error_handler.call(unknown_error, {:route => message.route, :payload => message.payload, :exchange_name => message.exchange_name, :options => message.options})
+                  end
+                end
+
+                # Do not requeue the message because something else horrible happened.
+                @current_messages = []
+
+                # TODO: Find a way to bubble this out of the thread for logging purposes.
+                # Reraise the error out of the publisher loop. The Supervisor will restart the consumer.
+                raise unknown_error
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_publisher/async/in_memory_adapter.rb

@@ -1,148 +1,54 @@
+require "active_publisher/message"
+require "active_publisher/async/in_memory_adapter/async_queue"
+require "active_publisher/async/in_memory_adapter/consumer_thread"
+require "multi_op_queue"
+
 module ActivePublisher
   module Async
-    class InMemoryAdapter
-      include ::ActivePublisher::Logging
-
-      attr_reader :async_queue
-
-      def initialize(drop_messages_when_queue_full = false, max_queue_size = 1_000_000, supervisor_interval = 0.2)
-        logger.info "Starting in-memory publisher adapter"
-
-        @async_queue = ::ActivePublisher::Async::InMemoryAdapter::AsyncQueue.new(
-          drop_messages_when_queue_full,
-          max_queue_size,
-          supervisor_interval
-        )
-      end
+    module InMemoryAdapter
 
-      def publish(route, payload, exchange_name, options = {})
-        message = ::ActivePublisher::Async::InMemoryAdapter::Message.new(route, payload, exchange_name, options)
-        async_queue.push(message)
-        nil
+      def self.new(*args)
+        ::ActivePublisher::Async::InMemoryAdapter::Adapter.new(*args)
       end
 
-      def shutdown!
-        max_wait_time = ::ActivePublisher.configuration.seconds_to_wait_for_graceful_shutdown
-        started_shutting_down_at = ::Time.now
+      class UnableToPersistMessageError < ::StandardError; end
 
-        logger.info "Draining async publisher in-memory adapter queue before shutdown. current queue size: #{async_queue.size}."
-        while async_queue.size > 0
-          if (::Time.now - started_shutting_down_at) > max_wait_time
-            logger.info "Forcing async publisher adapter shutdown because graceful shutdown period of #{max_wait_time} seconds was exceeded. Current queue size: #{async_queue.size}."
-            break
-          end
-
-          sleep 0.1
-        end
-      end
-
-      class AsyncQueue
+      class Adapter
         include ::ActivePublisher::Logging
 
-        attr_accessor :drop_messages_when_queue_full,
-                      :max_queue_size,
-                      :supervisor_interval
-
-        attr_reader :consumer, :queue, :supervisor
-
-        if ::RUBY_PLATFORM == "java"
-          NETWORK_ERRORS = [::MarchHare::Exception, ::Java::ComRabbitmqClient::AlreadyClosedException, ::Java::JavaIo::IOException].freeze
-        else
-          NETWORK_ERRORS = [::Bunny::Exception, ::Timeout::Error, ::IOError].freeze
-        end
-
-        def initialize(drop_messages_when_queue_full, max_queue_size, supervisor_interval)
-          @drop_messages_when_queue_full = drop_messages_when_queue_full
-          @max_queue_size = max_queue_size
-          @supervisor_interval = supervisor_interval
-          @queue = ::Queue.new
-          create_and_supervise_consumer!
-        end
-
-        def push(message)
-          # default of 1_000_000 messages
-          if queue.size > max_queue_size
-            # Drop messages if the queue is full and we were configured to do so
-            return if drop_messages_when_queue_full
+        attr_reader :async_queue
 
-            # By default we will raise an error to push the responsibility onto the caller
-            fail ::ActivePublisher::Async::InMemoryAdapter::UnableToPersistMessageError, "Queue is full, messages will be dropped."
-          end
+        def initialize(drop_messages_when_queue_full = false, max_queue_size = 1_000_000, supervisor_interval = 0.2)
+          logger.info "Starting in-memory publisher adapter"
 
-          queue.push(message)
+          @async_queue = ::ActivePublisher::Async::InMemoryAdapter::AsyncQueue.new(
+            drop_messages_when_queue_full,
+            max_queue_size,
+            supervisor_interval
+          )
         end
 
-        def size
-          queue.size
+        def publish(route, payload, exchange_name, options = {})
+          message = ::ActivePublisher::Message.new(route, payload, exchange_name, options)
+          async_queue.push(message)
+          nil
         end
 
-      private
+        def shutdown!
+          max_wait_time = ::ActivePublisher.configuration.seconds_to_wait_for_graceful_shutdown
+          started_shutting_down_at = ::Time.now
 
-        def await_network_reconnect
-          sleep ::ActivePublisher::RabbitConnection::NETWORK_RECOVERY_INTERVAL
-        end
-
-        def create_and_supervise_consumer!
-          @consumer = create_consumer
-          @supervisor = ::Thread.new do
-            loop do
-              unless consumer.alive?
-                # We might need to requeue the last message.
-                queue.push(@current_message) unless @current_message.nil?
-                consumer.kill
-                @consumer = create_consumer
-              end
-
-              # Pause before checking the consumer again.
-              sleep supervisor_interval
+          logger.info "Draining async publisher in-memory adapter queue before shutdown. current queue size: #{async_queue.size}."
+          while async_queue.size > 0
+            if (::Time.now - started_shutting_down_at) > max_wait_time
+              logger.info "Forcing async publisher adapter shutdown because graceful shutdown period of #{max_wait_time} seconds was exceeded. Current queue size: #{async_queue.size}."
+              break
             end
-          end
-        end
-
-        def create_consumer
-          ::Thread.new do
-            loop do
-              # Write "current_message" so we can requeue should something happen to the consumer.
-              @current_message = message = queue.pop
-
-              begin
-                ::ActivePublisher.publish(message.route, message.payload, message.exchange_name, message.options)
-
-                # Reset
-                @current_message = nil
-              rescue *NETWORK_ERRORS
-                # Sleep because connection is down
-                await_network_reconnect
 
-                # Requeue and try again.
-                queue.push(message)
-              rescue => unknown_error
-                # Do not requeue the message because something else horrible happened.
-                @current_message = nil
-
-                ::ActivePublisher.configuration.error_handler.call(unknown_error, {:route => message.route, :payload => message.payload, :exchange_name => message.exchange_name, :options => message.options})
-
-                # TODO: Find a way to bubble this out of the thread for logging purposes.
-                # Reraise the error out of the publisher loop. The Supervisor will restart the consumer.
-                raise unknown_error
-              end
-            end
+            sleep 0.1
           end
         end
-      end
-
-      class Message
-        attr_reader :route, :payload, :exchange_name, :options
-
-        def initialize(route, payload, exchange_name, options)
-          @route = route
-          @payload = payload
-          @exchange_name = exchange_name
-          @options = options
-        end
-      end
 
-      class UnableToPersistMessageError < ::StandardError
       end
     end
   end

data/lib/active_publisher/connection.rb

@@ -24,6 +24,8 @@ module ActivePublisher
 
         @connection = nil
       end
+    rescue Timeout::Error
+      # No-op ... this happens sometimes on MRI disconnect
     end
 
     # Private API

data/lib/active_publisher/message.rb

@@ -0,0 +1,3 @@
+module ActivePublisher
+  class Message < Struct.new(:route, :payload, :exchange_name, :options); end
+end

data/lib/active_publisher/version.rb

@@ -1,3 +1,3 @@
 module ActivePublisher
-  VERSION = "0.1.5"
+  VERSION = "0.2.0.pre"
 end

data/lib/active_publisher.rb

@@ -8,11 +8,15 @@ require "thread"
 require "active_publisher/logging"
 require "active_publisher/async"
 require "active_publisher/async/in_memory_adapter"
+require "active_publisher/message"
 require "active_publisher/version"
 require "active_publisher/configuration"
 require "active_publisher/connection"
 
 module ActivePublisher
+  class UnknownMessageClassError < StandardError; end
+  class ExchangeMismatchError < StandardError; end
+
   def self.configuration
     @configuration ||= ::ActivePublisher::Configuration.new
   end
@@ -33,6 +37,25 @@ module ActivePublisher
     end
   end
 
+  def self.publish_all(exchange_name, messages)
+    with_exchange(exchange_name) do |exchange|
+      loop do
+        break if messages.empty?
+        message = messages.shift
+
+        fail ActivePublisher::UnknownMessageClassError, "bulk publish messages must be ActivePublisher::Message" unless message.is_a?(ActivePublisher::Message)
+        fail ActivePublisher::ExchangeMismatchError, "bulk publish messages must match publish_all exchange_name" if message.exchange_name != exchange_name
+
+        begin
+          exchange.publish(message.payload, publishing_options(message.route, message.options || {}))
+        rescue
+          messages << message
+          raise
+        end
+      end
+    end
+  end
+
   def self.publishing_options(route, in_options = {})
     options = {
       :mandatory => false,

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: active_publisher
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.2.0.pre
 platform: java
 authors:
 - Brian Stien
@@ -12,7 +12,7 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2016-05-09 00:00:00.000000000 Z
+date: 2016-11-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -28,6 +28,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.7'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.2
+  name: multi_op_queue
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.2
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -109,9 +123,12 @@ files:
 - lib/active_publisher.rb
 - lib/active_publisher/async.rb
 - lib/active_publisher/async/in_memory_adapter.rb
+- lib/active_publisher/async/in_memory_adapter/async_queue.rb
+- lib/active_publisher/async/in_memory_adapter/consumer_thread.rb
 - lib/active_publisher/configuration.rb
 - lib/active_publisher/connection.rb
 - lib/active_publisher/logging.rb
+- lib/active_publisher/message.rb
 - lib/active_publisher/version.rb
 homepage: https://github.com/mxenabled/active_publisher
 licenses:
@@ -128,12 +145,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.4
+rubygems_version: 2.6.7
 signing_key:
 specification_version: 4
 summary: Aims to make publishing work across MRI and jRuby painless and add some nice features like automatially publishing lifecycle events for ActiveRecord models.