euston-daemons 1.0.0-java

data/lib/euston-daemons/message_buffer_daemon/config/environment.rb

@@ -0,0 +1,26 @@
+require_rel '../lib'
+
+Safely::Strategy::Log.logger = EUSTON_LOG
+Cqrs.uuid = Uuid
+AMQP.settings.merge! ErbYaml.read(AMQP_CONFIG_PATH, EUSTON_ENV)
+Euston::MessageBufferDaemon::Settings.configure ErbYaml.read(DAEMON_CONFIG_PATH, EUSTON_ENV)
+
+hash = ErbYaml.read(MONGOID_CONFIG_PATH, EUSTON_ENV)
+
+event_connection = Mongo::Connection.new hash[:host], hash[:port], :safe => hash[:safe] #, :logger => EUSTON_LOG
+read_connection  = Mongo::Connection.new hash[:host], hash[:port], :safe => hash[:safe] #, :logger => EUSTON_LOG
+
+event_database = hash[:event_store_database]
+
+Cqrs::RabbitMq.event_store_mongodb = Mongo::DB.new event_database, event_connection
+Cqrs::RabbitMq.read_model_mongodb = Mongo::DB.new hash[:read_model_database], read_connection
+
+EventStore::Persistence::Mongodb::Config.instance.logger = EUSTON_LOG
+EventStore::Persistence::Mongodb::Config.instance.database = event_database
+
+Cqrs::RabbitMq.event_store = EventStore::Persistence::Mongodb::MongoPersistenceFactory.build
+Cqrs::RabbitMq.event_store.init
+
+Cqrs::Repository.event_store = EventStore::OptimisticEventStore.new Cqrs::RabbitMq.event_store
+
+

data/lib/euston-daemons/message_buffer_daemon/lib/components/buffer_component.rb

@@ -0,0 +1,74 @@
+module Euston
+  module MessageBufferDaemon
+    class BufferComponent
+      include Euston::Daemons::ComponentShutdown
+
+      class << self
+        def command_component pub_channel, sub_channel
+          self.new Publisher.commands_buffer(pub_channel), Subscriber.commands_buffer(sub_channel)
+        end
+
+        def event_component pub_channel, sub_channel
+          self.new Publisher.events_buffer(pub_channel), Subscriber.events_buffer(sub_channel)
+        end
+      end
+
+      attr_reader :pub_thread, :sub_thread
+      attr_writer :wait_time, :subscribe_timeout
+
+      def initialize pub_buffer, sub_buffer
+        @pub_buffer = pub_buffer
+        @sub_buffer = sub_buffer
+      end
+
+      def wait_time
+        @wait_time ||= 0.2
+      end
+
+      def subscribe_timeout
+        @subscribe_timeout ||= 2000
+      end
+
+      def thread_state
+        {:publish => @pub_thread.status, :subscribe => @sub_thread.status}
+      end
+
+      def stop
+        @pub_thread[:stop], @sub_thread[:stop] = true, true
+        @pub_buffer.disconnect
+        @sub_buffer.disconnect
+      end
+
+      def start
+        @pub_thread = Thread.new do
+          until Thread.current[:stop] do
+            begin
+              @msg_dispatched_count = 0
+              begin
+                @msg_dispatched_count = @pub_buffer.dispatch_due_messages
+                check_exception_and_shutdown
+              end until Thread.current[:stop] || @msg_dispatched_count.zero?
+            rescue => e
+              Thread.current[:exception] = e
+            end
+            check_exception_and_shutdown
+            sleep self.wait_time
+          end
+        end
+
+        @sub_thread = Thread.new do
+          until Thread.current[:stop] do
+            begin
+              @sub_buffer.dequeue_buffered_messages(self.subscribe_timeout)
+              check_exception_and_shutdown
+            rescue => e
+              Thread.current[:exception] = e
+            end
+            check_exception_and_shutdown
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/euston-daemons/message_buffer_daemon/lib/components/event_store_component.rb

@@ -0,0 +1,51 @@
+module Euston
+  module MessageBufferDaemon
+    class EventStoreComponent
+      include Euston::Daemons::ComponentShutdown
+
+      attr_reader :cli_thread
+      attr_writer :wait_time
+
+      def initialize channel
+        @channel = channel
+      end
+
+      def wait_time
+        @wait_time ||= 0.2
+      end
+
+      def start
+        @cli_thread = Thread.new do
+          @event_buffer = Euston::MessageBufferDaemon::ReadModel::MessageBuffer.events
+          @event_store = Cqrs::RabbitMq.event_store
+          until Thread.current[:stop] do
+            begin
+              loop do
+                commits = @event_store.get_undispatched_commits
+                break if commits.empty?
+                commits.each do |commit|
+                  commit.events.each { |event| @event_buffer.buffer_new_message event.to_hash }
+                  @event_store.mark_commit_as_dispatched commit
+                end
+              end
+            rescue => e
+              Thread.current[:exception] = e
+            end
+            check_exception_and_shutdown
+            sleep self.wait_time
+          end
+        end
+      end
+
+      def thread_state
+        @cli_thread.status
+      end
+
+      def stop
+        @cli_thread[:stop] = true
+        @channel.disconnect
+      end
+
+    end
+  end
+end

data/lib/euston-daemons/message_buffer_daemon/lib/daemon.rb

@@ -0,0 +1,48 @@
+module Euston
+  module MessageBufferDaemon
+    class Daemon
+      include Euston::Daemon
+
+      attr_reader :clients, :queue
+
+      def initialize()
+        @queue = Queue.new
+
+        @clients = {
+          :commands_logger        => Euston::Daemons::BasicComponent.new(MessageLogger.commands_logger AMQP::Channel.new),
+          :events_logger          => Euston::Daemons::BasicComponent.new(MessageLogger.events_logger AMQP::Channel.new),
+          :command_buffer_handler => BufferComponent.command_component(AMQP::Channel.new, AMQP::Channel.new),
+          :event_buffer_handler   => BufferComponent.event_component(AMQP::Channel.new, AMQP::Channel.new),
+          :event_store_dispatcher => EventStoreComponent.new(AMQP::Channel.new)
+        }
+      end
+
+      def run
+        Thread.abort_on_exception = true
+
+        @clients.each do |name, component|
+          EUSTON_LOG.debug("Starting component: #{name}")
+          component.daemon = self
+          component.start
+          sleep 0.350
+        end
+
+        @clients.each do |name, component|
+          EUSTON_LOG.debug("Thread state of #{name}: #{component.thread_state.inspect}")
+        end
+
+        EUSTON_LOG.debug "Components started"
+
+        @queue.pop #<-------- stops here until interrupted
+
+        @clients.each do |name, component|
+          EUSTON_LOG.debug "Stopping component: #{name}"
+          component.stop
+        end
+
+        report_shutdown_reasons
+        EUSTON_LOG.debug "Components stopped"
+      end
+    end
+  end
+end

data/lib/euston-daemons/message_buffer_daemon/lib/message_logger.rb

@@ -0,0 +1,54 @@
+module Euston
+  class MessageLogger
+    class << self
+      def commands_logger channel
+        self.new channel, :commands
+      end
+
+      def events_logger channel
+        self.new channel, :events
+      end
+    end
+
+    include Cqrs::RabbitMq::Exchanges
+    include Cqrs::RabbitMq::Queues
+
+    def initialize channel, exchange_name
+      @channel = channel
+      @exchange = get_exchange channel, exchange_name
+      @read_model = Euston::MessageBufferDaemon::ReadModel::MessageLog.send exchange_name
+
+      @queue = get_queue channel, "#{exchange_name}_log"
+      @queue.bind @exchange, :routing_key => "#{exchange_name}.#"
+
+      @queue.when(:message_decode_failed => method(:log_failure),
+                  :message_failed        => method(:message_failed),
+                  :message_received      => method(:write_message_to_log))
+    end
+
+    def message_failed(message, error, header)
+      log_failure message, error
+      header.ack
+    end
+
+    def disconnect
+      @channel.disconnect
+    end
+
+    def start
+      @queue.safe_subscribe
+    end
+
+    def log_failure message, error
+      text = "A log queue subscription failed. [Error] #{error.message} [Payload] #{message}"
+      err = Cqrs::RabbitMq::MessageDecodeFailedError.new text
+      err.set_backtrace error.backtrace
+
+      Safely.report! err
+    end
+
+    def write_message_to_log message
+      @read_model.log_new_message message
+    end
+  end
+end

data/lib/euston-daemons/message_buffer_daemon/lib/publisher.rb

@@ -0,0 +1,56 @@
+module Euston
+  module MessageBufferDaemon
+    class Publisher
+      class << self
+        def commands_buffer channel
+          self.new channel, :commands
+        end
+
+        def events_buffer channel
+          self.new channel, :events
+        end
+      end
+
+      include Cqrs::RabbitMq::Exchanges
+
+      def initialize channel, exchange_name
+        @read_model = Euston::MessageBufferDaemon::ReadModel::MessageBuffer.send exchange_name
+
+        @channel = channel
+        @exchange = get_exchange @channel, exchange_name
+      end
+
+      def disconnect
+        @channel.disconnect
+      end
+
+      def dispatch_one_due_message
+        if (message = @read_model.find_next_message)
+          @read_model.set_next_attempt message
+          protected_publish(message)
+        end
+      end
+
+      def dispatch_due_messages
+        dispatched = 0
+        @read_model.find_due_messages.each do |message|
+          @read_model.set_next_attempt message
+          break unless protected_publish(message)
+          dispatched += 1
+        end
+        dispatched
+      end
+
+      def protected_publish(message)
+        ret = true
+        begin
+          @exchange.publish message['json'], default_publish_options.merge(:routing_key => "#{@exchange.name}.#{message['type']}")
+        rescue NativeException => e
+          Thread.current[:exception] = e
+          ret = false
+        end
+        ret
+      end
+    end
+  end
+end

data/lib/euston-daemons/message_buffer_daemon/lib/read_model/message_buffer.rb

@@ -0,0 +1,54 @@
+module Euston
+  module MessageBufferDaemon
+    module ReadModel
+      class MessageBuffer
+        class << self
+          def commands
+            self.new "buffered_commands", Cqrs::RabbitMq.read_model_mongodb
+          end
+
+          def events
+            self.new "buffered_events", Cqrs::RabbitMq.event_store_mongodb
+          end
+        end
+
+        def initialize name, mongodb
+          mongodb.create_collection name unless mongodb.collection_names.include? name
+
+          @collection = mongodb.collection name
+          @collection.ensure_index [ ['next_attempt', Mongo::ASCENDING] ], :unique => false, :name => "#{name}_next_attempt_index"
+        end
+
+        def buffer_new_message message
+          @collection.save({ 'doc_id' => message[:headers][:id],
+                             'type' => message[:headers][:type],
+                             'next_attempt' => (Time.now.to_f - 1.0),
+                             'json' => ActiveSupport::JSON.encode(message) }, :safe => { :fsync => true })
+        end
+
+        def find_next_message
+          query = { 'next_attempt' => { '$lte' => Time.now.to_f } }
+          @collection.find_one(query)
+        end
+
+        def find_due_messages
+          query = { 'next_attempt' => { '$lte' => Time.now.to_f }}
+          @collection.find(query)
+        end
+
+        def get_by_id id
+          @collection.find_one({ 'doc_id' => id }) #doc or nil
+        end
+
+        def set_next_attempt message
+          @collection.update({ 'doc_id' => message['doc_id'] },
+                             { '$set' => { 'next_attempt' => Time.now.to_f + 10 } }, :safe => { :fsync => true })
+        end
+
+        def remove_published_message id
+          @collection.remove({ 'doc_id' => id }, :safe => { :fsync => true })
+        end
+      end
+    end
+  end
+end

data/lib/euston-daemons/message_buffer_daemon/lib/read_model/message_log.rb

@@ -0,0 +1,36 @@
+module Euston
+  module MessageBufferDaemon
+    module ReadModel
+      class MessageLog
+        class << self
+          def commands
+            self.new "command_log", Cqrs::RabbitMq.read_model_mongodb
+          end
+
+          def events
+            self.new "event_log", Cqrs::RabbitMq.event_store_mongodb
+          end
+        end
+
+        def initialize name, mongodb
+          mongodb.create_collection name unless mongodb.collection_names.include? name
+
+          @collection = mongodb.collection name
+          @collection.ensure_index [ ['timestamp', Mongo::ASCENDING] ], :unique => false, :name => "#{name}_timestamp_index"
+        end
+
+        def find_all
+          @collection.find({}, { :sort => [ 'timestamp', Mongo::DESCENDING ] })
+        end
+
+        def log_new_message message
+          @collection.save({ '_id' => message[:headers][:id],
+                             'type' => message[:headers][:type],
+                             'version' => message[:headers][:version],
+                             'timestamp' => Time.now.to_f,
+                             'json' => ActiveSupport::JSON.encode(message) }, :safe => { :fsync => true })
+        end
+      end
+    end
+  end
+end

data/lib/euston-daemons/message_buffer_daemon/lib/settings.rb

@@ -0,0 +1,14 @@
+module Euston
+  module MessageBufferDaemon
+    module Settings
+      def self.configure(cfg=nil)
+        @config ||= {}
+        @config.merge!(cfg) if cfg && cfg.is_a?(Hash)
+      end
+
+      def self.debug
+        @config[:debug]
+      end
+    end
+  end
+end

data/lib/euston-daemons/message_buffer_daemon/lib/subscriber.rb

@@ -0,0 +1,60 @@
+module Euston
+  module MessageBufferDaemon
+    class Subscriber
+      class << self
+        def commands_buffer channel
+          self.new channel, :commands
+        end
+
+        def events_buffer channel
+          self.new channel, :events
+        end
+      end
+
+      include Cqrs::RabbitMq::Exchanges
+      include Cqrs::RabbitMq::Queues
+      include Hollywood
+
+      def initialize channel, exchange_name
+        @read_model = Euston::MessageBufferDaemon::ReadModel::MessageBuffer.send exchange_name
+
+        @channel = channel
+        @channel.prefetch(1)
+        @exchange = get_exchange @channel, exchange_name
+        @queue = get_queue @channel, "#{exchange_name}_buffer"
+        @queue.bind @exchange, :routing_key => "#{exchange_name}.#"
+
+        @queue.when(:message_decode_failed => method(:log_failure),
+                    :message_failed        => method(:message_failed),
+                    :message_received      => method(:remove_message_from_buffer))
+
+        @consumer = @queue.consumer
+      end
+
+      def disconnect
+        @channel.disconnect
+      end
+
+      def dequeue_buffered_messages(timeout) #will block until message or timeout
+        @queue.safe_subscribe_with_timeout(@consumer, timeout)
+      end
+
+      def log_failure message, error
+        text = "A buffer queue subscription failed. [Error] #{error.message} [Payload] #{message}"
+        err = Cqrs::RabbitMq::MessageDecodeFailedError.new text
+        err.set_backtrace error.backtrace
+
+        Safely.report! err
+      end
+
+      def message_failed(message, error, header)
+        header.ack!
+        log_failure message, error
+      end
+
+      def remove_message_from_buffer message
+        @read_model.remove_published_message message[:headers][:id]
+      end
+    end
+  end
+end