action_subscriber 2.0.1-java → 2.1.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 939204017ecb3b17925a0ceca1442798b98ddcdc
-  data.tar.gz: 1780324c82d3dac0a2f0d36d00bfaa724296782f
+  metadata.gz: a8a1a10bb43fef34f63038536e121540df818b14
+  data.tar.gz: 78e17fea12571a3c07037a7123f2e529ee1580a3
 SHA512:
-  metadata.gz: 258b3dcf397203650f2fe7c5b6590ddc3cf5ce0c9e81d1352d3a50505fc58ffa9b0dbcb5ec16b929d3ae9bb41580e6c15ba993da71194873eb4104ab58f44ba1
-  data.tar.gz: 9235bf4d8f54c8616ff77f958da778d04cd4a3f6271968a74f35245ad9fbf9a52c9b7cf1cf642e04dfd71a5e3a7ea8d586d7986f06a3fd6f6e1fc94ac2ab8e86
+  metadata.gz: a1392baccf77a3b65c695fcb1ff5234a1ba2ef38ec83f4400f43b617be6659f0b5b31d6348e95b84e6f89bf5279643d10bcae072c4023c0d94531d8d28108e5a
+  data.tar.gz: c55665bfb5d487f800251d50f3737e46143092c539b81b041ffb49d5f50adf7a5a2aaade0dabc7fe3bf659f20f070b53103097958d32cfee766ffee64f76baa6

data/lib/action_subscriber.rb

@@ -23,11 +23,13 @@ require "action_subscriber/bunny/subscriber"
 require "action_subscriber/march_hare/subscriber"
 require "action_subscriber/babou"
 require "action_subscriber/publisher"
+require "action_subscriber/publisher/async"
 require "action_subscriber/route"
 require "action_subscriber/route_set"
 require "action_subscriber/router"
 require "action_subscriber/threadpool"
 require "action_subscriber/base"
+require "action_subscriber/uri"
 
 module ActionSubscriber
   ##
@@ -110,6 +112,9 @@ module ActionSubscriber
   # Initialize config object
   config
 
+  # Intialize async publisher adapter
+  ::ActionSubscriber::Publisher::Async.publisher_adapter
+
   ::ActiveSupport.run_load_hooks(:action_subscriber, Base)
 
   ##
@@ -135,5 +140,6 @@ end
 require "action_subscriber/railtie" if defined?(Rails)
 
 at_exit do
+  ::ActionSubscriber::Publisher::Async.publisher_adapter.shutdown!
   ::ActionSubscriber::RabbitConnection.publisher_disconnect!
 end

data/lib/action_subscriber/configuration.rb

@@ -1,6 +1,10 @@
 module ActionSubscriber
   class Configuration
     attr_accessor :allow_low_priority_methods,
+                  :async_publisher,
+                  :async_publisher_drop_messages_when_queue_full,
+                  :async_publisher_max_queue_size,
+                  :async_publisher_supervisor_interval,
                   :decoder,
                   :default_exchange,
                   :error_handler,
@@ -8,17 +12,24 @@ module ActionSubscriber
                   :host,
                   :hosts,
                   :mode,
+                  :password,
                   :pop_interval,
                   :port,
                   :prefetch,
                   :publisher_confirms,
                   :seconds_to_wait_for_graceful_shutdown,
+                  :username,
                   :threadpool_size,
                   :timeout,
-                  :times_to_pop
+                  :times_to_pop,
+                  :virtual_host
 
     DEFAULTS = {
       :allow_low_priority_methods => false,
+      :async_publisher => 'memory',
+      :async_publisher_drop_messages_when_queue_full => false,
+      :async_publisher_max_queue_size => 1_000_000,
+      :async_publisher_supervisor_interval => 200, # in milliseconds
       :default_exchange => 'events',
       :heartbeat => 5,
       :host => 'localhost',
@@ -31,7 +42,10 @@ module ActionSubscriber
       :seconds_to_wait_for_graceful_shutdown => 30,
       :threadpool_size => 8,
       :timeout => 1,
-      :times_to_pop => 8
+      :times_to_pop => 8,
+      :username => "guest",
+      :password => "guest",
+      :virtual_host => "/"
     }
 
     ##
@@ -81,6 +95,13 @@ module ActionSubscriber
       self.decoder.merge!(decoders)
     end
 
+    def connection_string=(url)
+      settings = ::ActionSubscriber::URI.parse_amqp_url(url)
+      settings.each do |key, value|
+        send("#{key}=", value)
+      end
+    end
+
     def hosts
       return @hosts if @hosts.size > 0
       [ host ]

data/lib/action_subscriber/publisher/async.rb

@@ -0,0 +1,31 @@
+module ActionSubscriber
+  module Publisher
+    # Publish a message asynchronously to RabbitMQ.
+    #
+    # Asynchronous is designed to do two things:
+    # 1. Introduce the idea of a durable retry should the RabbitMQ connection disconnect.
+    # 2. Provide a higher-level pattern for fire-and-forget publishing.
+    #
+    # @param [String] route The routing key to use for this message.
+    # @param [String] payload The message you are sending. Should already be encoded as a string.
+    # @param [String] exchange The exchange you want to publish to.
+    # @param [Hash] options hash to set message parameters (e.g. headers).
+    def self.publish_async(route, payload, exchange_name, options = {})
+      Async.publisher_adapter.publish(route, payload, exchange_name, options)
+    end
+
+    module Async
+      def self.publisher_adapter
+        @publisher_adapter ||= case ::ActionSubscriber.configuration.async_publisher
+                               when /memory/i then
+                                 require "action_subscriber/publisher/async/in_memory_adapter"
+                                 InMemoryAdapter.new
+                               when /redis/i then
+                                 fail "Not yet implemented"
+                               else
+                                 fail "Unknown adapter '#{::ActionSubscriber.configuration.async_publisher}' provided"
+                               end
+      end
+    end
+  end
+end

data/lib/action_subscriber/publisher/async/in_memory_adapter.rb

@@ -0,0 +1,153 @@
+require "thread"
+
+module ActionSubscriber
+  module Publisher
+    module Async
+      class InMemoryAdapter
+        include ::ActionSubscriber::Logging
+
+        attr_reader :async_queue
+
+        def initialize
+          logger.info "Starting in-memory publisher adapter."
+
+          @async_queue = AsyncQueue.new
+        end
+
+        def publish(route, payload, exchange_name, options = {})
+          message = Message.new(route, payload, exchange_name, options)
+          async_queue.push(message)
+          nil
+        end
+
+        def shutdown!
+          max_wait_time = ::ActionSubscriber.configuration.seconds_to_wait_for_graceful_shutdown
+          started_shutting_down_at = ::Time.now
+
+          logger.info "Draining async publisher in-memory adapter queue before shutdown. Current queue size: #{async_queue.size}."
+          while async_queue.size > 0
+            if (::Time.now - started_shutting_down_at) > max_wait_time
+              logger.info "Forcing async publisher adapter shutdown because graceful shutdown period of #{max_wait_time} seconds was exceeded. Current queue size: #{async_queue.size}."
+              break
+            end
+
+            sleep 0.1
+          end
+        end
+
+        class Message
+          attr_reader :route, :payload, :exchange_name, :options
+
+          def initialize(route, payload, exchange_name, options)
+            @route = route
+            @payload = payload
+            @exchange_name = exchange_name
+            @options = options
+          end
+        end
+
+        class UnableToPersistMessageError < ::StandardError
+        end
+
+        class AsyncQueue
+          include ::ActionSubscriber::Logging
+
+          attr_reader :consumer, :queue, :supervisor
+
+          if ::RUBY_PLATFORM == "java"
+            NETWORK_ERRORS = [::MarchHare::Exception, ::Java::ComRabbitmqClient::AlreadyClosedException, ::Java::JavaIo::IOException].freeze
+          else
+            NETWORK_ERRORS = [::Bunny::Exception, ::Timeout::Error, ::IOError].freeze
+          end
+
+          def initialize
+            @queue = ::Queue.new
+            create_and_supervise_consumer!
+          end
+
+          def push(message)
+            # Default of 1_000_000 messages.
+            if queue.size > ::ActionSubscriber.configuration.async_publisher_max_queue_size
+              # Drop Messages if the queue is full and we were configured to do so.
+              return if ::ActionSubscriber.configuration.async_publisher_drop_messages_when_queue_full
+
+              # By default we will raise an error to push the responsibility onto the caller.
+              fail UnableToPersistMessageError, "Queue is full, messages will be dropped."
+            end
+
+            queue.push(message)
+          end
+
+          def size
+            queue.size
+          end
+
+        private
+
+          def await_network_reconnect
+            sleep ::ActionSubscriber::RabbitConnection::NETWORK_RECOVERY_INTERVAL
+          end
+
+          def create_and_supervise_consumer!
+            @consumer = create_consumer
+            @supervisor = ::Thread.new do
+              loop do
+                unless consumer.alive?
+                  # Why might need to requeue the last message.
+                  queue.push(@current_message) if @current_message.present?
+                  consumer.kill
+                  @consumer = create_consumer
+                end
+
+                # Pause before checking the consumer again.
+                sleep supervisor_interval
+              end
+            end
+          end
+
+          def create_consumer
+            ::Thread.new do
+              loop do
+                # Write "current_message" so we can requeue should something happen to the consumer. I don't love this, but it's
+                # better than writing my own `#peek' method.
+                @current_message = message = queue.pop
+
+                begin
+                  ::ActionSubscriber::Publisher.publish(message.route, message.payload, message.exchange_name, message.options)
+
+                  # Reset
+                  @current_message = nil
+                rescue *NETWORK_ERRORS
+                  # Sleep because the connection is down.
+                  await_network_reconnect
+
+                  # Requeue and try again.
+                  queue.push(message)
+                rescue => unknown_error
+                  # Do not requeue the message because something else horrible happened.
+                  @current_message = nil
+
+                  # Log the error.
+                  logger.info unknown_error.class
+                  logger.info unknown_error.message
+                  logger.info unknown_error.backtrace.join("\n")
+
+                  # TODO: Find a way to bubble this out of the thread for logging purposes.
+                  # Reraise the error out of the publisher loop. The Supervisor will restart the consumer.
+                  raise unknown_error
+                end
+              end
+            end
+          end
+
+          def supervisor_interval
+            @supervisor_interval ||= begin
+              interval_in_milliseconds = ::ActionSubscriber.configuration.async_publisher_supervisor_interval
+              interval_in_milliseconds / 1000.0
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/action_subscriber/rabbit_connection.rb

@@ -4,6 +4,7 @@ module ActionSubscriber
   module RabbitConnection
     SUBSCRIBER_CONNECTION_MUTEX = ::Mutex.new
     PUBLISHER_CONNECTION_MUTEX = ::Mutex.new
+    NETWORK_RECOVERY_INTERVAL = 1.freeze
 
     def self.publisher_connected?
       publisher_connection.try(:connected?)
@@ -61,12 +62,15 @@ module ActionSubscriber
 
     def self.connection_options
       {
+        :continuation_timeout          => ::ActionSubscriber.configuration.timeout * 1_000.0, #convert sec to ms
         :heartbeat                     => ::ActionSubscriber.configuration.heartbeat,
         :hosts                         => ::ActionSubscriber.configuration.hosts,
+        :pass                          => ::ActionSubscriber.configuration.password,
         :port                          => ::ActionSubscriber.configuration.port,
-        :continuation_timeout          => ::ActionSubscriber.configuration.timeout * 1_000.0, #convert sec to ms
+        :user                          => ::ActionSubscriber.configuration.username,
+        :vhost                         => ::ActionSubscriber.configuration.virtual_host,
         :automatically_recover         => true,
-        :network_recovery_interval     => 1,
+        :network_recovery_interval     => NETWORK_RECOVERY_INTERVAL,
         :recover_from_connection_close => true,
       }
     end

data/lib/action_subscriber/uri.rb

@@ -0,0 +1,28 @@
+# Taken and adapted from https://github.com/ruby-amqp/amq-protocol/blob/master/lib/amq/uri.rb
+require "cgi"
+require "uri"
+
+module ActionSubscriber
+  class URI
+    AMQP_PORTS = {"amqp" => 5672, "amqps" => 5671}.freeze
+
+    def self.parse_amqp_url(connection_string)
+      uri = ::URI.parse(connection_string)
+      raise ArgumentError.new("Connection URI must use amqp or amqps schema (example: amqp://bus.megacorp.internal:5766), learn more at http://bit.ly/ks8MXK") unless %w{amqp amqps}.include?(uri.scheme)
+
+      opts = {}
+
+      opts[:username]   = ::CGI::unescape(uri.user) if uri.user
+      opts[:password]   = ::CGI::unescape(uri.password) if uri.password
+      opts[:host]   = uri.host if uri.host
+      opts[:port]   = uri.port || AMQP_PORTS[uri.scheme]
+
+      if uri.path =~ %r{^/(.*)}
+        raise ArgumentError.new("#{uri} has multiple-segment path; please percent-encode any slashes in the vhost name (e.g. /production => %2Fproduction). Learn more at http://bit.ly/amqp-gem-and-connection-uris") if $1.index('/')
+        opts[:virtual_host] = ::CGI::unescape($1)
+      end
+
+      opts
+    end
+  end
+end

data/lib/action_subscriber/version.rb

@@ -1,3 +1,3 @@
 module ActionSubscriber
-  VERSION = "2.0.1"
+  VERSION = "2.1.0"
 end

data/spec/lib/action_subscriber/configuration_spec.rb

@@ -1,6 +1,10 @@
 describe ::ActionSubscriber::Configuration do
   describe "default values" do
     specify { expect(subject.allow_low_priority_methods).to eq(false) }
+    specify { expect(subject.async_publisher).to eq("memory") }
+    specify { expect(subject.async_publisher_drop_messages_when_queue_full).to eq(false) }
+    specify { expect(subject.async_publisher_max_queue_size).to eq(1_000_000) }
+    specify { expect(subject.async_publisher_supervisor_interval).to eq(200) }
     specify { expect(subject.default_exchange).to eq("events") }
     specify { expect(subject.heartbeat).to eq(5) }
     specify { expect(subject.host).to eq("localhost") }
@@ -34,4 +38,15 @@ describe ::ActionSubscriber::Configuration do
       }.to raise_error(/The foo decoder was given with arity of 2/)
     end
   end
+
+  describe "connection_string" do
+    it "explodes the connection string into the corresponding settings" do
+      subject.connection_string = "amqp://user:pass@host:100/vhost"
+      expect(subject.username).to eq("user")
+      expect(subject.password).to eq("pass")
+      expect(subject.host).to eq("host")
+      expect(subject.port).to eq(100)
+      expect(subject.virtual_host).to eq("vhost")
+    end
+  end
 end

data/spec/lib/action_subscriber/publisher/async/in_memory_adapter_spec.rb

@@ -0,0 +1,135 @@
+describe ::ActionSubscriber::Publisher::Async::InMemoryAdapter do
+  let(:route) { "test" }
+  let(:payload) { "message" }
+  let(:exchange_name) { "place" }
+  let(:options) { { :test => :ok } }
+  let(:message) { described_class::Message.new(route, payload, exchange_name, options) }
+  let(:mock_queue) { double(:push => nil, :size => 0) }
+
+  describe "#publish" do
+    before do
+      allow(described_class::Message).to receive(:new).with(route, payload, exchange_name, options).and_return(message)
+      allow(described_class::AsyncQueue).to receive(:new).and_return(mock_queue)
+    end
+
+    it "can publish a message to the queue" do
+      expect(mock_queue).to receive(:push).with(message)
+      subject.publish(route, payload, exchange_name, options)
+    end
+  end
+
+  describe "#shutdown!" do
+    # This is called when the rspec finishes. I'm sure we can make this a better test.
+  end
+
+  describe "::ActionSubscriber::Publisher::Async::InMemoryAdapter::Message" do
+    specify { expect(message.route).to eq(route) }
+    specify { expect(message.payload).to eq(payload) }
+    specify { expect(message.exchange_name).to eq(exchange_name) }
+    specify { expect(message.options).to eq(options) }
+  end
+
+  describe "::ActionSubscriber::Publisher::Async::InMemoryAdapter::AsyncQueue" do
+    subject { described_class::AsyncQueue.new }
+
+    describe ".initialize" do
+      it "creates a supervisor" do
+        expect_any_instance_of(described_class::AsyncQueue).to receive(:create_and_supervise_consumer!)
+        subject
+      end
+    end
+
+    describe "#create_and_supervise_consumer!" do
+      it "creates a supervisor" do
+        expect_any_instance_of(described_class::AsyncQueue).to receive(:create_consumer)
+        subject
+      end
+
+      it "restarts the consumer when it dies" do
+        consumer = subject.consumer
+        consumer.kill
+
+        verify_expectation_within(0.1) do
+          expect(consumer).to_not be_alive
+        end
+
+        verify_expectation_within(0.3) do
+          expect(subject.consumer).to be_alive
+        end
+      end
+    end
+
+    describe "#create_consumer" do
+      it "can successfully publish a message" do
+        expect(::ActionSubscriber::Publisher).to receive(:publish).with(route, payload, exchange_name, options)
+        subject.push(message)
+        sleep 0.1 # Await results
+      end
+
+      context "when network error occurs" do
+        let(:error) { described_class::AsyncQueue::NETWORK_ERRORS.first }
+        before { allow(::ActionSubscriber::Publisher).to receive(:publish).and_raise(error) }
+
+        it "requeues the message" do
+          consumer = subject.consumer
+          expect(consumer).to be_alive
+          expect(subject).to receive(:await_network_reconnect).at_least(:once)
+          subject.push(message)
+          sleep 0.1 # Await results
+        end
+      end
+
+      context "when an unknown error occurs" do
+        before { allow(::ActionSubscriber::Publisher).to receive(:publish).and_raise(ArgumentError) }
+
+        it "kills the consumer" do
+          consumer = subject.consumer
+          expect(consumer).to be_alive
+          subject.push(message)
+          sleep 0.1 # Await results
+          expect(consumer).to_not be_alive
+        end
+      end
+    end
+
+    describe "#push" do
+      after { ::ActionSubscriber.configuration.async_publisher_max_queue_size = 1000 }
+      after { ::ActionSubscriber.configuration.async_publisher_drop_messages_when_queue_full = false }
+
+      context "when the queue has room" do
+        before { allow(::Queue).to receive(:new).and_return(mock_queue) }
+
+        it "successfully adds to the queue" do
+          expect(mock_queue).to receive(:push).with(message)
+          subject.push(message)
+        end
+      end
+
+      context "when the queue is full" do
+        before { ::ActionSubscriber.configuration.async_publisher_max_queue_size = -1 }
+
+        context "and we're dropping messages" do
+          before { ::ActionSubscriber.configuration.async_publisher_drop_messages_when_queue_full = true }
+
+          it "adding to the queue should not raise an error" do
+            expect { subject.push(message) }.to_not raise_error
+          end
+        end
+
+        context "and we're not dropping messages" do
+          before { ::ActionSubscriber.configuration.async_publisher_drop_messages_when_queue_full = false }
+
+          it "adding to the queue should raise error back to caller" do
+            expect { subject.push(message) }.to raise_error(described_class::UnableToPersistMessageError)
+          end
+        end
+      end
+    end
+
+    describe "#size" do
+      it "can return the size of the queue" do
+        expect(subject.size).to eq(0)
+      end
+    end
+  end
+end

data/spec/lib/action_subscriber/publisher/async_spec.rb

@@ -0,0 +1,40 @@
+describe ::ActionSubscriber::Publisher::Async do
+
+  before { described_class.instance_variable_set(:@publisher_adapter, nil) }
+  after { ::ActionSubscriber.configuration.async_publisher = "memory" }
+
+  let(:mock_adapter) { double(:publish => nil) }
+
+  describe ".publish_async" do
+    before { allow(described_class).to receive(:publisher_adapter).and_return(mock_adapter) }
+
+    it "calls through the adapter" do
+      expect(mock_adapter).to receive(:publish).with("1", "2", "3", { "four" => "five" })
+      ::ActionSubscriber::Publisher.publish_async("1", "2", "3", { "four" => "five" })
+    end
+  end
+
+  context "when an in-memory adapter is selected" do
+    before { ::ActionSubscriber.configuration.async_publisher = "memory" }
+
+    it "Creates an in-memory publisher" do
+      expect(described_class.publisher_adapter).to be_an(::ActionSubscriber::Publisher::Async::InMemoryAdapter)
+    end
+  end
+
+  context "when an redis adapter is selected" do
+    before { ::ActionSubscriber.configuration.async_publisher = "redis" }
+
+    it "raises an error" do
+      expect { described_class.publisher_adapter }.to raise_error("Not yet implemented")
+    end
+  end
+
+  context "when some random adapter is selected" do
+    before { ::ActionSubscriber.configuration.async_publisher = "yolo" }
+
+    it "raises an error" do
+      expect { described_class.publisher_adapter }.to raise_error("Unknown adapter 'yolo' provided")
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -13,6 +13,7 @@ require 'support/user_subscriber'
 require 'action_subscriber/rspec'
 
 # Silence the Logger
+$TESTING = true
 ::ActionSubscriber::Logging.initialize_logger(nil)
 
 RSpec.configure do |config|

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: action_subscriber
 version: !ruby/object:Gem::Version
-  version: 2.0.1
+  version: 2.1.0
 platform: java
 authors:
 - Brian Stien
@@ -12,7 +12,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-01-26 00:00:00.000000000 Z
+date: 2016-02-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -213,6 +213,8 @@ files:
 - lib/action_subscriber/middleware/router.rb
 - lib/action_subscriber/middleware/runner.rb
 - lib/action_subscriber/publisher.rb
+- lib/action_subscriber/publisher/async.rb
+- lib/action_subscriber/publisher/async/in_memory_adapter.rb
 - lib/action_subscriber/rabbit_connection.rb
 - lib/action_subscriber/railtie.rb
 - lib/action_subscriber/route.rb
@@ -221,6 +223,7 @@ files:
 - lib/action_subscriber/rspec.rb
 - lib/action_subscriber/subscribable.rb
 - lib/action_subscriber/threadpool.rb
+- lib/action_subscriber/uri.rb
 - lib/action_subscriber/version.rb
 - routing.md
 - spec/integration/around_filters_spec.rb
@@ -244,6 +247,8 @@ files:
 - spec/lib/action_subscriber/middleware/error_handler_spec.rb
 - spec/lib/action_subscriber/middleware/router_spec.rb
 - spec/lib/action_subscriber/middleware/runner_spec.rb
+- spec/lib/action_subscriber/publisher/async/in_memory_adapter_spec.rb
+- spec/lib/action_subscriber/publisher/async_spec.rb
 - spec/lib/action_subscriber/publisher_spec.rb
 - spec/lib/action_subscriber/router_spec.rb
 - spec/lib/action_subscriber/subscribable_spec.rb
@@ -270,7 +275,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.5.0
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: ActionSubscriber is a DSL that allows a rails app to consume messages from a RabbitMQ broker.
@@ -296,6 +301,8 @@ test_files:
 - spec/lib/action_subscriber/middleware/error_handler_spec.rb
 - spec/lib/action_subscriber/middleware/router_spec.rb
 - spec/lib/action_subscriber/middleware/runner_spec.rb
+- spec/lib/action_subscriber/publisher/async/in_memory_adapter_spec.rb
+- spec/lib/action_subscriber/publisher/async_spec.rb
 - spec/lib/action_subscriber/publisher_spec.rb
 - spec/lib/action_subscriber/router_spec.rb
 - spec/lib/action_subscriber/subscribable_spec.rb