action_subscriber 2.0.1 → 2.1.0.pre1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fa7ffc8e113cf3414e95bd9ff2f4271716316da3
-  data.tar.gz: bb9f4b158454c734e75484f8d3fb722ea9817b3d
+  metadata.gz: cdfd01dd3dd2055815f165b70bce923d36fe1db7
+  data.tar.gz: 261530f967fe4368afa58295344e1fe40ee70746
 SHA512:
-  metadata.gz: e3495e8deb06c7ec7540a3cfb0f8080badcbfaa6565e9f1839bdf515c6c557997727e60802f95bca40683db7b5b98d2338eff316575f609c3d2b8616e3503bee
-  data.tar.gz: 2a34bf987ad18fa9b5d8fd852c8148173dbcc5b0bbf94329cd89ccbead8d66c152df1d3a766b9badfbf5baf5ce988a3c99e5c857cb82dc5f152fe8b2e075e661
+  metadata.gz: 109f19c08d6975bca0af2df1ff70483ef8e3211d235e4c4767cba1d386c78085be51d021357ccc76f1d2f86564fa3a43c3500869ba690e970f5cc06a3032ed9e
+  data.tar.gz: 3646c5fe3efb28a73d98a3a8aa991dfeac31e71d7946b0549dc7e32610c30cd6914241e6775f867828729e3bb15c6fa15ba48be715a1b1a7bc30352acc6e160e

data/lib/action_subscriber/configuration.rb

@@ -1,6 +1,10 @@
 module ActionSubscriber
   class Configuration
     attr_accessor :allow_low_priority_methods,
+                  :async_publisher,
+                  :async_publisher_drop_messages_when_queue_full,
+                  :async_publisher_max_queue_size,
+                  :async_publisher_supervisor_interval,
                   :decoder,
                   :default_exchange,
                   :error_handler,
@@ -19,6 +23,10 @@ module ActionSubscriber
 
     DEFAULTS = {
       :allow_low_priority_methods => false,
+      :async_publisher => 'memory',
+      :async_publisher_drop_messages_when_queue_full => false,
+      :async_publisher_max_queue_size => 1_000_000,
+      :async_publisher_supervisor_interval => 200, # in milliseconds
       :default_exchange => 'events',
       :heartbeat => 5,
       :host => 'localhost',

data/lib/action_subscriber/publisher/async/in_memory_adapter.rb

@@ -0,0 +1,153 @@
+require "thread"
+
+module ActionSubscriber
+  module Publisher
+    module Async
+      class InMemoryAdapter
+        include ::ActionSubscriber::Logging
+
+        attr_reader :async_queue
+
+        def initialize
+          logger.info "Starting in-memory publisher adapter."
+
+          @async_queue = AsyncQueue.new
+        end
+
+        def publish(route, payload, exchange_name, options = {})
+          message = Message.new(route, payload, exchange_name, options)
+          async_queue.push(message)
+          nil
+        end
+
+        def shutdown!
+          max_wait_time = ::ActionSubscriber.configuration.seconds_to_wait_for_graceful_shutdown
+          started_shutting_down_at = ::Time.now
+
+          logger.info "Draining async publisher in-memory adapter queue before shutdown. Current queue size: #{async_queue.size}."
+          while async_queue.size > 0
+            if (::Time.now - started_shutting_down_at) > max_wait_time
+              logger.info "Forcing async publisher adapter shutdown because graceful shutdown period of #{max_wait_time} seconds was exceeded. Current queue size: #{async_queue.size}."
+              break
+            end
+
+            sleep 0.1
+          end
+        end
+
+        class Message
+          attr_reader :route, :payload, :exchange_name, :options
+
+          def initialize(route, payload, exchange_name, options)
+            @route = route
+            @payload = payload
+            @exchange_name = exchange_name
+            @options = options
+          end
+        end
+
+        class UnableToPersistMessageError < ::StandardError
+        end
+
+        class AsyncQueue
+          include ::ActionSubscriber::Logging
+
+          attr_reader :consumer, :queue, :supervisor
+
+          if ::RUBY_PLATFORM == "java"
+            NETWORK_ERRORS = [::MarchHare::Exception, ::Java::ComRabbitmqClient::AlreadyClosedException, ::Java::JavaIo::IOException].freeze
+          else
+            NETWORK_ERRORS = [::Bunny::Exception, ::Timeout::Error, ::IOError].freeze
+          end
+
+          def initialize
+            @queue = ::Queue.new
+            create_and_supervise_consumer!
+          end
+
+          def push(message)
+            # Default of 1_000_000 messages.
+            if queue.size > ::ActionSubscriber.configuration.async_publisher_max_queue_size
+              # Drop Messages if the queue is full and we were configured to do so.
+              return if ::ActionSubscriber.configuration.async_publisher_drop_messages_when_queue_full
+
+              # By default we will raise an error to push the responsibility onto the caller.
+              fail UnableToPersistMessageError, "Queue is full, messages will be dropped."
+            end
+
+            queue.push(message)
+          end
+
+          def size
+            queue.size
+          end
+
+        private
+
+          def await_network_reconnect
+            sleep ::ActionSubscriber::RabbitConnection::NETWORK_RECOVERY_INTERVAL
+          end
+
+          def create_and_supervise_consumer!
+            @consumer = create_consumer
+            @supervisor = ::Thread.new do
+              loop do
+                unless consumer.alive?
+                  # Why might need to requeue the last message.
+                  queue.push(@current_message) if @current_message.present?
+                  consumer.kill
+                  @consumer = create_consumer
+                end
+
+                # Pause before checking the consumer again.
+                sleep supervisor_interval
+              end
+            end
+          end
+
+          def create_consumer
+            ::Thread.new do
+              loop do
+                # Write "current_message" so we can requeue should something happen to the consumer. I don't love this, but it's
+                # better than writing my own `#peek' method.
+                @current_message = message = queue.pop
+
+                begin
+                  ::ActionSubscriber::Publisher.publish(message.route, message.payload, message.exchange_name, message.options)
+
+                  # Reset
+                  @current_message = nil
+                rescue *NETWORK_ERRORS
+                  # Sleep because the connection is down.
+                  await_network_reconnect
+
+                  # Requeue and try again.
+                  queue.push(message)
+                rescue => unknown_error
+                  # Do not requeue the message because something else horrible happened.
+                  @current_message = nil
+
+                  # Log the error.
+                  logger.info unknown_error.class
+                  logger.info unknown_error.message
+                  logger.info unknown_error.backtrace.join("\n")
+
+                  # TODO: Find a way to bubble this out of the thread for logging purposes.
+                  # Reraise the error out of the publisher loop. The Supervisor will restart the consumer.
+                  raise unknown_error
+                end
+              end
+            end
+          end
+
+          def supervisor_interval
+            @supervisor_interval ||= begin
+              interval_in_milliseconds = ::ActionSubscriber.configuration.async_publisher_supervisor_interval
+              interval_in_milliseconds / 1000.0
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/action_subscriber/publisher/async.rb

@@ -0,0 +1,31 @@
+module ActionSubscriber
+  module Publisher
+    # Publish a message asynchronously to RabbitMQ.
+    #
+    # Asynchronous is designed to do two things:
+    # 1. Introduce the idea of a durable retry should the RabbitMQ connection disconnect.
+    # 2. Provide a higher-level pattern for fire-and-forget publishing.
+    #
+    # @param [String] route The routing key to use for this message.
+    # @param [String] payload The message you are sending. Should already be encoded as a string.
+    # @param [String] exchange The exchange you want to publish to.
+    # @param [Hash] options hash to set message parameters (e.g. headers).
+    def self.publish_async(route, payload, exchange_name, options = {})
+      Async.publisher_adapter.publish(route, payload, exchange_name, options)
+    end
+
+    module Async
+      def self.publisher_adapter
+        @publisher_adapter ||= case ::ActionSubscriber.configuration.async_publisher
+                               when /memory/i then
+                                 require "action_subscriber/publisher/async/in_memory_adapter"
+                                 InMemoryAdapter.new
+                               when /redis/i then
+                                 fail "Not yet implemented"
+                               else
+                                 fail "Unknown adapter '#{::ActionSubscriber.configuration.async_publisher}' provided"
+                               end
+      end
+    end
+  end
+end

data/lib/action_subscriber/rabbit_connection.rb

@@ -4,6 +4,7 @@ module ActionSubscriber
   module RabbitConnection
     SUBSCRIBER_CONNECTION_MUTEX = ::Mutex.new
     PUBLISHER_CONNECTION_MUTEX = ::Mutex.new
+    NETWORK_RECOVERY_INTERVAL = 1.freeze
 
     def self.publisher_connected?
       publisher_connection.try(:connected?)
@@ -66,7 +67,7 @@ module ActionSubscriber
         :port                          => ::ActionSubscriber.configuration.port,
         :continuation_timeout          => ::ActionSubscriber.configuration.timeout * 1_000.0, #convert sec to ms
         :automatically_recover         => true,
-        :network_recovery_interval     => 1,
+        :network_recovery_interval     => NETWORK_RECOVERY_INTERVAL,
         :recover_from_connection_close => true,
       }
     end

data/lib/action_subscriber/version.rb

@@ -1,3 +1,3 @@
 module ActionSubscriber
-  VERSION = "2.0.1"
+  VERSION = "2.1.0.pre1"
 end

data/lib/action_subscriber.rb

@@ -23,6 +23,7 @@ require "action_subscriber/bunny/subscriber"
 require "action_subscriber/march_hare/subscriber"
 require "action_subscriber/babou"
 require "action_subscriber/publisher"
+require "action_subscriber/publisher/async"
 require "action_subscriber/route"
 require "action_subscriber/route_set"
 require "action_subscriber/router"
@@ -110,6 +111,9 @@ module ActionSubscriber
   # Initialize config object
   config
 
+  # Intialize async publisher adapter
+  ::ActionSubscriber::Publisher::Async.publisher_adapter
+
   ::ActiveSupport.run_load_hooks(:action_subscriber, Base)
 
   ##
@@ -135,5 +139,6 @@ end
 require "action_subscriber/railtie" if defined?(Rails)
 
 at_exit do
+  ::ActionSubscriber::Publisher::Async.publisher_adapter.shutdown!
   ::ActionSubscriber::RabbitConnection.publisher_disconnect!
 end

data/spec/lib/action_subscriber/configuration_spec.rb

@@ -1,7 +1,10 @@
 describe ::ActionSubscriber::Configuration do
   describe "default values" do
     specify { expect(subject.allow_low_priority_methods).to eq(false) }
-    specify { expect(subject.default_exchange).to eq("events") }
+    specify { expect(subject.async_publisher).to eq("memory") }
+    specify { expect(subject.async_publisher_drop_messages_when_queue_full).to eq(false) }
+    specify { expect(subject.async_publisher_max_queue_size).to eq(1_000_000) }
+    specify { expect(subject.async_publisher_supervisor_interval).to eq(200) }
     specify { expect(subject.heartbeat).to eq(5) }
     specify { expect(subject.host).to eq("localhost") }
     specify { expect(subject.mode).to eq('subscribe') }

data/spec/lib/action_subscriber/publisher/async/in_memory_adapter_spec.rb

@@ -0,0 +1,135 @@
+describe ::ActionSubscriber::Publisher::Async::InMemoryAdapter do
+  let(:route) { "test" }
+  let(:payload) { "message" }
+  let(:exchange_name) { "place" }
+  let(:options) { { :test => :ok } }
+  let(:message) { described_class::Message.new(route, payload, exchange_name, options) }
+  let(:mock_queue) { double(:push => nil, :size => 0) }
+
+  describe "#publish" do
+    before do
+      allow(described_class::Message).to receive(:new).with(route, payload, exchange_name, options).and_return(message)
+      allow(described_class::AsyncQueue).to receive(:new).and_return(mock_queue)
+    end
+
+    it "can publish a message to the queue" do
+      expect(mock_queue).to receive(:push).with(message)
+      subject.publish(route, payload, exchange_name, options)
+    end
+  end
+
+  describe "#shutdown!" do
+    # This is called when the rspec finishes. I'm sure we can make this a better test.
+  end
+
+  describe "::ActionSubscriber::Publisher::Async::InMemoryAdapter::Message" do
+    specify { expect(message.route).to eq(route) }
+    specify { expect(message.payload).to eq(payload) }
+    specify { expect(message.exchange_name).to eq(exchange_name) }
+    specify { expect(message.options).to eq(options) }
+  end
+
+  describe "::ActionSubscriber::Publisher::Async::InMemoryAdapter::AsyncQueue" do
+    subject { described_class::AsyncQueue.new }
+
+    describe ".initialize" do
+      it "creates a supervisor" do
+        expect_any_instance_of(described_class::AsyncQueue).to receive(:create_and_supervise_consumer!)
+        subject
+      end
+    end
+
+    describe "#create_and_supervise_consumer!" do
+      it "creates a supervisor" do
+        expect_any_instance_of(described_class::AsyncQueue).to receive(:create_consumer)
+        subject
+      end
+
+      it "restarts the consumer when it dies" do
+        consumer = subject.consumer
+        consumer.kill
+
+        verify_expectation_within(0.1) do
+          expect(consumer).to_not be_alive
+        end
+
+        verify_expectation_within(0.3) do
+          expect(subject.consumer).to be_alive
+        end
+      end
+    end
+
+    describe "#create_consumer" do
+      it "can successfully publish a message" do
+        expect(::ActionSubscriber::Publisher).to receive(:publish).with(route, payload, exchange_name, options)
+        subject.push(message)
+        sleep 0.1 # Await results
+      end
+
+      context "when network error occurs" do
+        let(:error) { described_class::AsyncQueue::NETWORK_ERRORS.first }
+        before { allow(::ActionSubscriber::Publisher).to receive(:publish).and_raise(error) }
+
+        it "requeues the message" do
+          consumer = subject.consumer
+          expect(consumer).to be_alive
+          expect(subject).to receive(:await_network_reconnect).at_least(:once)
+          subject.push(message)
+          sleep 0.1 # Await results
+        end
+      end
+
+      context "when an unknown error occurs" do
+        before { allow(::ActionSubscriber::Publisher).to receive(:publish).and_raise(ArgumentError) }
+
+        it "kills the consumer" do
+          consumer = subject.consumer
+          expect(consumer).to be_alive
+          subject.push(message)
+          sleep 0.1 # Await results
+          expect(consumer).to_not be_alive
+        end
+      end
+    end
+
+    describe "#push" do
+      after { ::ActionSubscriber.configuration.async_publisher_max_queue_size = 1000 }
+      after { ::ActionSubscriber.configuration.async_publisher_drop_messages_when_queue_full = false }
+
+      context "when the queue has room" do
+        before { allow(::Queue).to receive(:new).and_return(mock_queue) }
+
+        it "successfully adds to the queue" do
+          expect(mock_queue).to receive(:push).with(message)
+          subject.push(message)
+        end
+      end
+
+      context "when the queue is full" do
+        before { ::ActionSubscriber.configuration.async_publisher_max_queue_size = -1 }
+
+        context "and we're dropping messages" do
+          before { ::ActionSubscriber.configuration.async_publisher_drop_messages_when_queue_full = true }
+
+          it "adding to the queue should not raise an error" do
+            expect { subject.push(message) }.to_not raise_error
+          end
+        end
+
+        context "and we're not dropping messages" do
+          before { ::ActionSubscriber.configuration.async_publisher_drop_messages_when_queue_full = false }
+
+          it "adding to the queue should raise error back to caller" do
+            expect { subject.push(message) }.to raise_error(described_class::UnableToPersistMessageError)
+          end
+        end
+      end
+    end
+
+    describe "#size" do
+      it "can return the size of the queue" do
+        expect(subject.size).to eq(0)
+      end
+    end
+  end
+end

data/spec/lib/action_subscriber/publisher/async_spec.rb

@@ -0,0 +1,40 @@
+describe ::ActionSubscriber::Publisher::Async do
+
+  before { described_class.instance_variable_set(:@publisher_adapter, nil) }
+  after { ::ActionSubscriber.configuration.async_publisher = "memory" }
+
+  let(:mock_adapter) { double(:publish => nil) }
+
+  describe ".publish_async" do
+    before { allow(described_class).to receive(:publisher_adapter).and_return(mock_adapter) }
+
+    it "calls through the adapter" do
+      expect(mock_adapter).to receive(:publish).with("1", "2", "3", { "four" => "five" })
+      ::ActionSubscriber::Publisher.publish_async("1", "2", "3", { "four" => "five" })
+    end
+  end
+
+  context "when an in-memory adapter is selected" do
+    before { ::ActionSubscriber.configuration.async_publisher = "memory" }
+
+    it "Creates an in-memory publisher" do
+      expect(described_class.publisher_adapter).to be_an(::ActionSubscriber::Publisher::Async::InMemoryAdapter)
+    end
+  end
+
+  context "when an redis adapter is selected" do
+    before { ::ActionSubscriber.configuration.async_publisher = "redis" }
+
+    it "raises an error" do
+      expect { described_class.publisher_adapter }.to raise_error("Not yet implemented")
+    end
+  end
+
+  context "when some random adapter is selected" do
+    before { ::ActionSubscriber.configuration.async_publisher = "yolo" }
+
+    it "raises an error" do
+      expect { described_class.publisher_adapter }.to raise_error("Unknown adapter 'yolo' provided")
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -13,6 +13,7 @@ require 'support/user_subscriber'
 require 'action_subscriber/rspec'
 
 # Silence the Logger
+$TESTING = true
 ::ActionSubscriber::Logging.initialize_logger(nil)
 
 RSpec.configure do |config|

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: action_subscriber
 version: !ruby/object:Gem::Version
-  version: 2.0.1
+  version: 2.1.0.pre1
 platform: ruby
 authors:
 - Brian Stien
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-26 00:00:00.000000000 Z
+date: 2016-02-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -214,6 +214,8 @@ files:
 - lib/action_subscriber/middleware/router.rb
 - lib/action_subscriber/middleware/runner.rb
 - lib/action_subscriber/publisher.rb
+- lib/action_subscriber/publisher/async.rb
+- lib/action_subscriber/publisher/async/in_memory_adapter.rb
 - lib/action_subscriber/rabbit_connection.rb
 - lib/action_subscriber/railtie.rb
 - lib/action_subscriber/route.rb
@@ -245,6 +247,8 @@ files:
 - spec/lib/action_subscriber/middleware/error_handler_spec.rb
 - spec/lib/action_subscriber/middleware/router_spec.rb
 - spec/lib/action_subscriber/middleware/runner_spec.rb
+- spec/lib/action_subscriber/publisher/async/in_memory_adapter_spec.rb
+- spec/lib/action_subscriber/publisher/async_spec.rb
 - spec/lib/action_subscriber/publisher_spec.rb
 - spec/lib/action_subscriber/router_spec.rb
 - spec/lib/action_subscriber/subscribable_spec.rb
@@ -266,12 +270,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: ActionSubscriber is a DSL that allows a rails app to consume messages from
@@ -298,6 +302,8 @@ test_files:
 - spec/lib/action_subscriber/middleware/error_handler_spec.rb
 - spec/lib/action_subscriber/middleware/router_spec.rb
 - spec/lib/action_subscriber/middleware/runner_spec.rb
+- spec/lib/action_subscriber/publisher/async/in_memory_adapter_spec.rb
+- spec/lib/action_subscriber/publisher/async_spec.rb
 - spec/lib/action_subscriber/publisher_spec.rb
 - spec/lib/action_subscriber/router_spec.rb
 - spec/lib/action_subscriber/subscribable_spec.rb