euston-rabbitmq 1.0.0

data/lib/euston-rabbitmq/exchanges.rb

@@ -0,0 +1,17 @@
+module Euston
+  module RabbitMq
+    module Exchanges
+      def default_exchange_options
+        { :durable => true, :nowait => false }
+      end
+
+      def default_publish_options
+        { :immediate => false, :mandatory => true, :persistent => true }
+      end
+
+      def get_exchange channel, name, opts = {}
+        channel.topic name.to_s, default_exchange_options.merge(opts)
+      end
+    end
+  end
+end

data/lib/euston-rabbitmq/handler_bindings.rb

@@ -0,0 +1,115 @@
+module Euston
+  module RabbitMq
+    class HandlerBindings
+      include Euston::RabbitMq::Exchanges
+      include Euston::RabbitMq::Queues
+
+      def initialize channel
+        @channel = channel
+        @exchanges = { :commands => get_exchange(@channel, :commands),
+                         :events => get_exchange(@channel, :events) }
+        @namespaces = []
+      end
+
+      def add_namespace namespace
+        @namespaces << namespace
+      end
+
+      def namespaced_handler_types
+        ret = []
+        @namespaces.each do |namespace|
+          namespace.constants.each do |handler_type|
+            ret << [namespace, handler_type]
+          end
+        end
+        ret
+      end
+
+      def finalize_bindings
+        namespaced_handler_types.each do |namespace, handler_type|
+          bind_handler handler_type
+        end
+      end
+
+      protected
+
+      def bind_handler handler_type
+        # virtual
+      end
+
+      def find_namespaces_containing_handler_type handler_type
+        @namespaces.select { |namespace| namespace.const_defined? handler_type }
+      end
+
+      def attach_queue_hook_listeners queue
+        queue.when(:message_decode_failed => method(:log_decode_failure),
+                   :message_failed        => method(:handle_failure),
+                   :message_received      => method(:call_handler))
+
+        queue.safe_subscribe
+      end
+
+      def call_handler(message)
+        # abstract
+      end
+
+      def create_message_failed_message error, failed_message, amqp_header
+        {
+          :headers =>
+          {
+            :id => Euston.uuid.generate.to_s,
+            :type => :message_failed,
+            :version => 1,
+            :timestamp => Time.now.to_f
+          },
+
+          :body =>
+          {
+            :message => failed_message,
+            :routing_key => amqp_header.method.routing_key,
+            :error => error.message,
+            :backtrace => error.backtrace
+          }
+        }
+      end
+
+      def handle_failure message, error, amqp_header
+        failures = message[:failures] || 0
+        failures = failures + 1
+        message[:failures] = failures
+
+        begin
+          if failures == 3
+            message.delete :failures
+            message = create_message_failed_message error, message, amqp_header
+
+            options = { :key => 'events.message_failed' }
+            exchange = :events
+          else
+            options = { :key => amqp_header.method.routing_key }
+            exchange = amqp_header.method.routing_key.split('.').first.to_sym
+          end
+
+          options = default_publish_options.merge options
+          exchange = @exchanges[exchange]
+          message = ::ActiveSupport::JSON.encode message
+
+          exchange.publish message, options
+
+          amqp_header.ack
+        rescue StandardError => e
+          amqp_header.reject :requeue => true
+          raise e
+        end
+      end
+
+      def log_decode_failure message, error
+        text = "A handler queue subscription failed. [Error] #{error.message} [Payload] #{message}"
+        err = Euston::RabbitMq::MessageDecodeFailedError.new text
+        err.set_backtrace error.backtrace
+
+        Safely.report! err
+      end
+    end
+  end
+end

data/lib/euston-rabbitmq/message_buffer.rb

@@ -0,0 +1,68 @@
+module Euston
+  module RabbitMq
+    class MessageBuffer
+      class << self
+        def commands_buffer channel = nil
+          @command_buffers ||= MessageBuffer.new channel, :commands
+        end
+
+        def events_buffer channel = nil
+          @event_buffers ||= MessageBuffer.new channel, :events
+        end
+      end
+
+      include Euston::RabbitMq::Exchanges
+      include Euston::RabbitMq::Queues
+      include Hollywood
+
+      def initialize channel, exchange_name
+
+        @channel = channel
+        @exchange = get_exchange channel, exchange_name
+        @read_model = Euston::RabbitMq::ReadModel::MessageBuffer.send exchange_name
+
+        @queue = get_queue channel, "#{exchange_name}_buffer"
+        @queue.bind @exchange, :routing_key => "#{exchange_name}.#"
+
+        @queue.when(:message_decode_failed => method(:log_failure),
+                    :message_failed        => method(:handle_failure),
+                    :message_received      => method(:remove_message_from_buffer))
+
+        @queue.safe_subscribe
+      end
+
+      def handle_failure(message, error, header)
+        log_failure message, error
+        header.ack
+      end
+
+      def dispatch_due_messages
+        dispatched = false
+
+        @read_model.find_due_messages.to_a.each do |message|
+          @read_model.set_next_attempt message
+          @exchange.publish message['json'], default_publish_options.merge(:routing_key => "#{@exchange.name}.#{message['type']}")
+          dispatched = true
+        end
+
+        callback(:no_messages_were_due) unless dispatched
+      end
+
+      def log_failure message, error
+        text = "A buffer queue subscription failed. [Error] #{error.message} [Payload] #{message}"
+        err = Euston::RabbitMq::MessageDecodeFailedError.new text
+        err.set_backtrace error.backtrace
+
+        Safely.report! err
+      end
+
+      def push message
+        @read_model.buffer_new_message message
+      end
+
+      def remove_message_from_buffer message
+        @read_model.remove_published_message message[:headers][:id]
+      end
+    end
+  end
+end

data/lib/euston-rabbitmq/message_logger.rb

@@ -0,0 +1,50 @@
+module Euston
+  module RabbitMq
+    class MessageLogger
+      class << self
+        def commands_logger channel = nil
+          @command_logger ||= MessageLogger.new channel, :commands
+        end
+
+        def events_logger channel = nil
+          @event_logger ||= MessageLogger.new channel, :events
+        end
+      end
+
+      include Euston::RabbitMq::Exchanges
+      include Euston::RabbitMq::Queues
+
+      def initialize channel, exchange_name
+        @channel = channel
+        @exchange = get_exchange channel, exchange_name
+        @read_model = Euston::RabbitMq::ReadModel::MessageLog.send exchange_name
+
+        @queue = get_queue channel, "#{exchange_name}_log"
+        @queue.bind @exchange, :routing_key => "#{exchange_name}.#"
+
+        @queue.when(:message_decode_failed => method(:log_failure),
+                    :message_failed        => method(:handle_failure),
+                    :message_received      => method(:write_message_to_log))
+
+        @queue.safe_subscribe
+      end
+
+      def handle_failure(message, error, header)
+        log_failure message, error
+        header.ack
+      end
+
+      def log_failure message, error
+        text = "A log queue subscription failed. [Error] #{error.message} [Payload] #{message}"
+        err = Euston::RabbitMq::MessageDecodeFailedError.new text
+        err.set_backtrace error.backtrace
+
+        Safely.report! err
+      end
+
+      def write_message_to_log message
+        @read_model.log_new_message message
+      end
+    end
+  end
+end

data/lib/euston-rabbitmq/queue.rb

@@ -0,0 +1,30 @@
+module AMQP
+  class Queue
+    include Hollywood
+
+    def safe_subscribe
+      cb = Hollywood.instance_method(:callback).bind self
+
+      self.subscribe :ack => true do |header, body|
+        rt = ::RobustThread.new(:args => [header, body], :label => "#{self.name} queue subscriber") do |header, body|
+          begin
+            message = ::ActiveSupport::JSON.decode body
+            message.recursive_symbolize_keys!
+
+            begin
+              cb.call :message_received, message
+              header.ack
+            rescue Euston::EventStore::ConcurrencyError
+              header.reject :requeue => true
+            rescue StandardError => e
+              cb.call :message_failed, message, e, header
+            end
+          rescue StandardError => e
+            cb.call :message_decode_failed, body, e
+            header.ack
+          end
+        end
+      end
+    end
+  end
+end

data/lib/euston-rabbitmq/queues.rb

@@ -0,0 +1,13 @@
+module Euston
+  module RabbitMq
+    module Queues
+      def default_queue_options
+        { :durable => true, :nowait => true }
+      end
+
+      def get_queue channel, name, opts = {}
+        channel.queue name.to_s, default_queue_options.merge(opts)
+      end
+    end
+  end
+end

data/lib/euston-rabbitmq/read_model/failed_message.rb

@@ -0,0 +1,36 @@
+module Euston
+  module RabbitMq
+    module ReadModel
+      class FailedMessage
+        def initialize
+          name = 'failed_messages'
+
+          mongodb = Euston::RabbitMq.event_store_mongodb
+          mongodb.create_collection name unless mongodb.collection_names.include? name
+
+          @collection = mongodb.collection name
+          @collection.ensure_index [ ['failure_timestamp', Mongo::ASCENDING] ], :unique => false, :name => 'failed_messages_failure_timestamp_index'
+        end
+
+        def get_by_id id
+          @collection.find_one({ '_id' => id })
+        end
+
+        def find_all
+          @collection.find({}, { :sort => [ 'failure_timestamp', Mongo::DESCENDING ] })
+        end
+
+        def log_failure failure
+          failure.recursive_stringify_keys!
+          failure['_id'] = failure.delete 'message_id'
+
+          @collection.save(failure, :safe => { :fsync => true })
+        end
+
+        def remove_by_id id
+          @collection.remove({ '_id' => id }, :safe => { :fsync => true })
+        end
+      end
+    end
+  end
+end

data/lib/euston-rabbitmq/read_model/message_buffer.rb

@@ -0,0 +1,52 @@
+module Euston
+  module RabbitMq
+    module ReadModel
+      class MessageBuffer
+        class << self
+          def commands
+            @commands ||= MessageBuffer.new "buffered_commands"
+          end
+
+          def events
+            @events ||= MessageBuffer.new "buffered_events"
+          end
+        end
+
+        def initialize name
+          mongodb = Euston::RabbitMq.event_store_mongodb
+          mongodb.create_collection name unless mongodb.collection_names.include? name
+
+          @collection = mongodb.collection name
+          @collection.ensure_index [ ['next_attempt', Mongo::ASCENDING] ], :unique => false, :name => "#{name}_next_attempt_index"
+        end
+
+        def buffer_new_message message
+          @collection.save({ '_id' => message[:headers][:id],
+                             'type' => message[:headers][:type],
+                             'next_attempt' => Time.now.to_f,
+                             'json' => ::ActiveSupport::JSON.encode(message) }, :safe => { :fsync => true })
+        end
+
+        def find_due_messages
+          query = { 'next_attempt' => { '$lte' => Time.now.to_f } }
+          order = [ 'next_attempt', Mongo::ASCENDING ]
+
+          @collection.find(query).sort(order)
+        end
+
+        def get_by_id id
+          @collection.find_one({ '_id' => id })
+        end
+
+        def set_next_attempt message
+          @collection.update({ '_id' => message['_id'] },
+                             { '$set' => { 'next_attempt' => message['next_attempt'] + 10 } }, :safe => { :fsync => true })
+        end
+
+        def remove_published_message id
+          @collection.remove({ '_id' => id }, :safe => { :fsync => true })
+        end
+      end
+    end
+  end
+end

data/lib/euston-rabbitmq/read_model/message_log.rb

@@ -0,0 +1,37 @@
+module Euston
+  module RabbitMq
+    module ReadModel
+      class MessageLog
+        class << self
+          def commands
+            @commands ||= MessageLog.new "command_log"
+          end
+
+          def events
+            @events ||= MessageLog.new "event_log"
+          end
+        end
+
+        def initialize name
+          mongodb = Euston::RabbitMq.event_store_mongodb
+          mongodb.create_collection name unless mongodb.collection_names.include? name
+
+          @collection = mongodb.collection name
+          @collection.ensure_index [ ['timestamp', Mongo::ASCENDING] ], :unique => false, :name => "#{name}_timestamp_index"
+        end
+
+        def find_all
+          @collection.find({}, { :sort => [ 'timestamp', Mongo::DESCENDING ] })
+        end
+
+        def log_new_message message
+          @collection.save({ '_id' => message[:headers][:id],
+                             'type' => message[:headers][:type],
+                             'version' => message[:headers][:version],
+                             'timestamp' => Time.now.to_f,
+                             'json' => ::ActiveSupport::JSON.encode(message) }, :safe => { :fsync => true })
+        end
+      end
+    end
+  end
+end

data/lib/euston-rabbitmq/version.rb

@@ -0,0 +1,5 @@
+module Euston
+  module RabbitMq
+    VERSION = "1.0.0"
+  end
+end

data/lib/euston-rabbitmq.rb

@@ -0,0 +1,28 @@
+require 'require_all'
+require 'active_support/core_ext/hash/keys'
+require 'active_support/core_ext/string/inflections'
+require 'active_support/json'
+require 'active_support/ordered_hash'
+require 'robustthread'
+
+if RUBY_PLATFORM == 'java'
+  require 'jessica'
+else
+  require 'eventmachine'
+  require 'amqp'
+end
+
+require 'hash-keys'
+require 'hollywood'
+require 'euston'
+require 'euston-eventstore'
+
+require_rel 'euston-rabbitmq'
+
+module Euston
+  module RabbitMq
+    class << self
+      attr_accessor :event_store, :event_store_mongodb, :read_model_mongodb
+    end
+  end
+end

data/spec/command_buffer_spec.rb

@@ -0,0 +1,69 @@
+require File.expand_path('../spec_helper.rb', __FILE__)
+
+unless RUBY_PLATFORM == 'java'
+  describe 'command buffer' do
+    include EustonRmqSpec
+
+    context 'the command buffer is created' do
+      include Euston::RabbitMq::Queues
+
+      amqp_before do
+        @buffer = Euston::RabbitMq::MessageBuffer.new @channel, :commands
+        @queue = get_queue @channel, :commands_buffer
+      end
+
+      it 'declares the commands buffer queue' do
+        done(0.5) {
+          @queue.name.should == 'commands_buffer'
+          @queue.opts.should include(:durable => true)
+        }
+      end
+
+      it 'binds the command buffer queue to the commands exchange with a wildcard routing key' do
+        received = []
+        data = Factory.command
+
+        delayed(1) {
+          exchange = @channel.find_exchange 'commands'
+          @queue.when(:message_received => ->(message) { received << message })
+          exchange.publish ::ActiveSupport::JSON.encode(data), :routing_key => "commands.#{Euston.uuid.generate}"
+        }
+
+        done(2) {
+          received.should have(1).item
+          received.first.should == data
+        }
+      end
+    end
+
+    context 'a command is pushed into the buffer by client code' do
+      amqp_before :each do
+        @buffer = Euston::RabbitMq::MessageBuffer.new @channel, :commands
+        @command = Factory.command
+        @buffer.push @command
+        @read_model = Euston::RabbitMq::ReadModel::MessageBuffer.new 'buffered_commands'
+      end
+
+      it 'writes the command to persistent storage' do
+        @read_model.find_due_messages.to_a.should have(1).items
+
+        due_command = @read_model.find_due_messages.first
+        due_command['_id'].should == @command[:headers][:id]
+        due_command['type'].should == @command[:headers][:type]
+        due_command['json'].should == ::ActiveSupport::JSON.encode(@command)
+
+        done(0.3)
+      end
+
+      it 'removes the command from the buffer storage when it is obtained from the buffer queue' do
+        delayed(1) {
+          @buffer.dispatch_due_messages
+        }
+
+        done(3) {
+          @read_model.get_by_id(@command[:headers][:id]).should be_nil
+        }
+      end
+    end
+  end
+end

data/spec/event_buffer_spec.rb

@@ -0,0 +1,69 @@
+require File.expand_path('../spec_helper.rb', __FILE__)
+
+unless RUBY_PLATFORM == 'java'
+  describe 'event buffer' do
+    include EustonRmqSpec
+
+    context 'the event buffer is created' do
+      include Euston::RabbitMq::Queues
+
+      amqp_before do
+        @buffer = Euston::RabbitMq::MessageBuffer.new @channel, :events
+        @queue = get_queue @channel, :events_buffer
+      end
+
+      it 'declares the events buffer queue' do
+        done(0.5) {
+          @queue.name.should == 'events_buffer'
+          @queue.opts.should include(:durable => true)
+        }
+      end
+
+      it 'binds the event buffer queue to the events exchange with a wildcard routing key' do
+        received = []
+        data = Factory.command
+
+        delayed(0.3) {
+          exchange = @channel.find_exchange 'events'
+          @queue.when(:message_received => ->(message) { received << message })
+          exchange.publish ::ActiveSupport::JSON.encode(data), :routing_key => "events.#{Euston.uuid.generate}"
+        }
+
+        done(2) {
+          received.should have(1).item
+          received.first.should == data
+        }
+      end
+    end
+
+    context 'an event is pushed into the buffer by client code' do
+      amqp_before :each do
+        @buffer = Euston::RabbitMq::MessageBuffer.new @channel, :events
+        @command = Factory.command
+        @buffer.push @command
+        @read_model = Euston::RabbitMq::ReadModel::MessageBuffer.new 'buffered_events'
+      end
+
+      it 'writes the event to persistent storage' do
+        @read_model.find_due_messages.to_a.should have(1).items
+
+        due_command = @read_model.find_due_messages.first
+        due_command['_id'].should == @command[:headers][:id]
+        due_command['type'].should == @command[:headers][:type]
+        due_command['json'].should == ::ActiveSupport::JSON.encode(@command)
+
+        done(0.3)
+      end
+
+      it 'removes the event from the buffer storage when it is obtained from the buffer queue' do
+        delayed(1) {
+          @buffer.dispatch_due_messages
+        }
+
+        done(3) {
+          @read_model.get_by_id(@command[:headers][:id]).should be_nil
+        }
+      end
+    end
+  end
+end

data/spec/exchange_declaration_spec.rb

@@ -0,0 +1,28 @@
+require File.expand_path('../spec_helper.rb', __FILE__)
+
+unless RUBY_PLATFORM == 'java'
+  describe 'exchange declaration' do
+    include EustonRmqSpec
+
+    context 'declaring the commands exchange' do
+      include Euston::RabbitMq::Exchanges
+      it 'declares the commands exchange' do
+        exchange = get_exchange @channel, :commands
+
+        done(0.3) {
+          exchange.name.should == 'commands'
+          exchange.should be_durable
+        }
+      end
+
+      it 'declares the events exchange' do
+        exchange = get_exchange @channel, :events
+
+        done(0.3) {
+          exchange.name.should == 'events'
+          exchange.should be_durable
+        }
+      end
+    end
+  end
+end