euston-daemons 1.0.1

data/lib/euston-daemons/event_processor_daemon/lib/settings.rb

@@ -0,0 +1,22 @@
+module Euston
+  module EventProcessorDaemon
+    module Settings
+      def self.configure(cfg=nil)
+        @config ||= {}
+        @config.merge!(cfg) if cfg && cfg.is_a?(Hash)
+      end
+      def self.server_instances
+        @config[:server_instances] || 2
+      end
+      def self.bus_port_base
+        (@config[:zmq_base_port] || 8200).to_i
+      end
+      def self.mongo_db_name
+        @config[:mongo_db_name]
+      end
+      def self.debug
+        @config[:debug]
+      end
+    end
+  end
+end

data/lib/euston-daemons/event_processor_daemon/rake_task.rb

@@ -0,0 +1,37 @@
+module Euston
+  class EventProcessorRakeTask < Euston::Daemons::RakeTask
+    attr_accessor :amqp_config_path, :daemon_config_path, :event_handler_namespaces, :i18n_locales_path, :mongoid_config_path
+
+    def initialize
+      @event_handler_namespaces = []
+      super(:event_processor_daemon)
+    end
+
+    def before_creating_task
+      @daemon_path = File.expand_path(File.dirname __FILE__) + File::SEPARATOR
+      @daemon_class = 'Euston::EventProcessorDaemon::Daemon'
+    end
+
+    def initialize_paths
+      EUSTON_LOG.debug "AMQP config path: #{@amqp_config_path}"
+      Object.const_set :AMQP_CONFIG_PATH, @amqp_config_path
+
+      EUSTON_LOG.debug "Daemon config path: #{@daemon_config_path}"
+      Object.const_set :DAEMON_CONFIG_PATH, @daemon_config_path
+
+      EUSTON_LOG.debug "Event handler namespaces: #{@event_handler_namespaces}"
+      Object.const_set :EVENT_HANDLER_NAMESPACES, @event_handler_namespaces
+
+      EUSTON_LOG.debug "i18n locales path: #{@i18n_locales_path}"
+      Object.const_set :I18N_LOCALES_PATH, @i18n_locales_path
+
+      EUSTON_LOG.debug "Mongoid config path: #{@mongoid_config_path}"
+      Object.const_set :MONGOID_CONFIG_PATH, @mongoid_config_path
+    end
+
+    def load_environment
+      EUSTON_LOG.debug "Loading environment"
+      require_rel 'config/environment.rb'
+    end
+  end
+end

data/lib/euston-daemons/framework/basic_component.rb

@@ -0,0 +1,33 @@
+module Euston
+  module Daemons
+    class BasicComponent
+      include Euston::Daemons::ComponentShutdown
+
+      attr_reader :client, :cli_thread
+
+      def initialize(hosted_object)
+        @client = hosted_object
+      end
+
+      def start
+        @cli_thread = Thread.new do
+          begin
+            @client.start
+          rescue => e
+            Thread.current[:exception] = e
+          end
+          check_exception_and_shutdown
+        end
+      end
+
+      def thread_state
+        @cli_thread.status
+      end
+
+      def stop
+        @cli_thread[:stop] = true
+        @client.disconnect if @client.respond_to?(:disconnect)
+      end
+    end
+  end
+end

data/lib/euston-daemons/framework/component_shutdown.rb

@@ -0,0 +1,22 @@
+module Euston
+  module Daemons
+    module ComponentShutdown
+      attr_writer :daemon
+
+      def check_exception_and_shutdown(errors=[])
+        if errors.compact.empty?
+          errors << Thread.current[:exception]
+          return if errors.compact.empty?
+        end
+        errors.compact.each do |e|
+          if e.respond_to?(:cause) #its a serious (Java?) exception
+            Thread.current[:stop] = true
+            @daemon.irrecoverable_signal_from_component! e
+          else
+            Safely.report! e
+          end
+        end
+      end
+    end
+  end
+end

data/lib/euston-daemons/framework/daemon.rb

@@ -0,0 +1,27 @@
+module Euston
+  module Daemon
+    def errors
+      @errors ||= Queue.new
+    end
+
+    def irrecoverable_signal_from_component!(e)
+      self.errors.push e
+      @queue.push "SHUTDOWN"
+    end
+
+    def report_shutdown_reasons
+      sleep(0.25)
+      reports = []
+      until self.errors.empty? do
+        e = self.errors.pop(true) rescue nil
+        next if e.nil?
+        case e
+        when NativeException
+          Safely.report!(e.cause)
+        else
+          Safely.report!(e)
+        end
+      end
+    end
+  end
+end

data/lib/euston-daemons/framework/queue.rb

@@ -0,0 +1,71 @@
+class RabbitMQClient
+  class Queue
+    include Hollywood
+
+    attr_writer :timeout
+
+    def delivery_timeout
+      @timeout ||= 500
+    end
+
+    def safe_subscribe
+      _consumer = self.consumer
+      until Thread.current[:stop] do
+        safe_subscribe_with_timeout(_consumer, self.delivery_timeout)
+      end
+    end
+
+    def safe_handle_message(msg)
+      begin
+        message = JSON.parse_sym(msg.body)
+        begin
+          callback :message_received, message
+          msg.ack!
+        rescue Euston::EventStore::ConcurrencyError
+          msg.reject! true #requeue
+        rescue => e
+          callback :message_failed, message, e, msg
+          Safely.report! e
+        end
+      rescue => e
+        callback :message_decode_failed, msg.body, e
+        msg.ack!
+        Safely.report! e
+      end
+    end
+
+    def consumer(auto_ack=false)
+        consumer = QueueingConsumer.new(@channel)
+        @channel.basic_consume(@name, auto_ack, consumer)
+        consumer
+    end
+
+    def safe_subscribe_with_timeout(consumer, timeout=500)
+      loop do
+        delivery = nil
+        begin
+          delivery = consumer.next_delivery(timeout)
+        rescue NativeException => e
+          Thread.current[:exception] = e
+          break
+        end
+        break if delivery.nil?
+        remsg = ReactiveMessage.new(@channel, delivery, String.from_java_bytes(delivery.get_body))
+        safe_handle_message remsg
+        @channel.basic_ack(delivery.envelope.delivery_tag, false) if remsg.should_acknowledge?
+      end
+    end
+  end
+
+  class ReactiveMessage
+    def reject(opts={})
+      self.reject!(opts.fetch(:requeue, true))
+    end
+    def ack(multiple=false)
+      self.ack!
+    end
+    def method()
+      self.envelope
+    end
+  end
+end

data/lib/euston-daemons/message_buffer_daemon/config/environment.rb

@@ -0,0 +1,30 @@
+require 'jessica'
+require 'euston'
+require 'euston-eventstore'
+require 'euston-rabbitmq'
+
+require_rel '../lib'
+
+Safely::Strategy::Log.logger = EUSTON_LOG
+AMQP.settings.merge! ErbYaml.read(AMQP_CONFIG_PATH, EUSTON_ENV)
+Euston::MessageBufferDaemon::Settings.configure ErbYaml.read(DAEMON_CONFIG_PATH, EUSTON_ENV)
+
+hash = ErbYaml.read(MONGOID_CONFIG_PATH, EUSTON_ENV)
+
+event_connection = Mongo::Connection.new hash[:host], hash[:port], :safe => hash[:safe] #, :logger => EUSTON_LOG
+read_connection  = Mongo::Connection.new hash[:host], hash[:port], :safe => hash[:safe] #, :logger => EUSTON_LOG
+
+event_database = hash[:event_store_database]
+
+Euston::RabbitMq.event_store_mongodb = Mongo::DB.new event_database, event_connection
+Euston::RabbitMq.read_model_mongodb = Mongo::DB.new hash[:read_model_database], read_connection
+
+Euston::EventStore::Persistence::Mongodb::Config.instance.logger = EUSTON_LOG
+Euston::EventStore::Persistence::Mongodb::Config.instance.database = event_database
+
+Euston::RabbitMq.event_store = Euston::EventStore::Persistence::Mongodb::MongoPersistenceFactory.build
+Euston::RabbitMq.event_store.init
+
+Euston::Repository.event_store = Euston::EventStore::OptimisticEventStore.new Euston::RabbitMq.event_store
+
+

data/lib/euston-daemons/message_buffer_daemon/lib/components/buffer_component.rb

@@ -0,0 +1,74 @@
+module Euston
+  module MessageBufferDaemon
+    class BufferComponent
+      include Euston::Daemons::ComponentShutdown
+
+      class << self
+        def command_component pub_channel, sub_channel
+          self.new Publisher.commands_buffer(pub_channel), Subscriber.commands_buffer(sub_channel)
+        end
+
+        def event_component pub_channel, sub_channel
+          self.new Publisher.events_buffer(pub_channel), Subscriber.events_buffer(sub_channel)
+        end
+      end
+
+      attr_reader :pub_thread, :sub_thread
+      attr_writer :wait_time, :subscribe_timeout
+
+      def initialize pub_buffer, sub_buffer
+        @pub_buffer = pub_buffer
+        @sub_buffer = sub_buffer
+      end
+
+      def wait_time
+        @wait_time ||= 0.2
+      end
+
+      def subscribe_timeout
+        @subscribe_timeout ||= 2000
+      end
+
+      def thread_state
+        {:publish => @pub_thread.status, :subscribe => @sub_thread.status}
+      end
+
+      def stop
+        @pub_thread[:stop], @sub_thread[:stop] = true, true
+        @pub_buffer.disconnect
+        @sub_buffer.disconnect
+      end
+
+      def start
+        @pub_thread = Thread.new do
+          until Thread.current[:stop] do
+            begin
+              @msg_dispatched_count = 0
+              begin
+                @msg_dispatched_count = @pub_buffer.dispatch_due_messages
+                check_exception_and_shutdown
+              end until Thread.current[:stop] || @msg_dispatched_count.zero?
+            rescue => e
+              Thread.current[:exception] = e
+            end
+            check_exception_and_shutdown
+            sleep self.wait_time
+          end
+        end
+
+        @sub_thread = Thread.new do
+          until Thread.current[:stop] do
+            begin
+              @sub_buffer.dequeue_buffered_messages(self.subscribe_timeout)
+              check_exception_and_shutdown
+            rescue => e
+              Thread.current[:exception] = e
+            end
+            check_exception_and_shutdown
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/euston-daemons/message_buffer_daemon/lib/components/event_store_component.rb

@@ -0,0 +1,51 @@
+module Euston
+  module MessageBufferDaemon
+    class EventStoreComponent
+      include Euston::Daemons::ComponentShutdown
+
+      attr_reader :cli_thread
+      attr_writer :wait_time
+
+      def initialize channel
+        @channel = channel
+      end
+
+      def wait_time
+        @wait_time ||= 0.2
+      end
+
+      def start
+        @cli_thread = Thread.new do
+          @event_buffer = Euston::MessageBufferDaemon::ReadModel::MessageBuffer.events
+          @event_store = Euston::RabbitMq.event_store
+          until Thread.current[:stop] do
+            begin
+              loop do
+                commits = @event_store.get_undispatched_commits
+                break if commits.empty?
+                commits.each do |commit|
+                  commit.events.each { |event| @event_buffer.buffer_new_message event.to_hash }
+                  @event_store.mark_commit_as_dispatched commit
+                end
+              end
+            rescue => e
+              Thread.current[:exception] = e
+            end
+            check_exception_and_shutdown
+            sleep self.wait_time
+          end
+        end
+      end
+
+      def thread_state
+        @cli_thread.status
+      end
+
+      def stop
+        @cli_thread[:stop] = true
+        @channel.disconnect
+      end
+
+    end
+  end
+end

data/lib/euston-daemons/message_buffer_daemon/lib/daemon.rb

@@ -0,0 +1,48 @@
+module Euston
+  module MessageBufferDaemon
+    class Daemon
+      include Euston::Daemon
+
+      attr_reader :clients, :queue
+
+      def initialize()
+        @queue = Queue.new
+
+        @clients = {
+          :commands_logger        => Euston::Daemons::BasicComponent.new(MessageLogger.commands_logger AMQP::Channel.new),
+          :events_logger          => Euston::Daemons::BasicComponent.new(MessageLogger.events_logger AMQP::Channel.new),
+          :command_buffer_handler => BufferComponent.command_component(AMQP::Channel.new, AMQP::Channel.new),
+          :event_buffer_handler   => BufferComponent.event_component(AMQP::Channel.new, AMQP::Channel.new),
+          :event_store_dispatcher => EventStoreComponent.new(AMQP::Channel.new)
+        }
+      end
+
+      def run
+        Thread.abort_on_exception = true
+
+        @clients.each do |name, component|
+          EUSTON_LOG.debug("Starting component: #{name}")
+          component.daemon = self
+          component.start
+          sleep 0.350
+        end
+
+        @clients.each do |name, component|
+          EUSTON_LOG.debug("Thread state of #{name}: #{component.thread_state.inspect}")
+        end
+
+        EUSTON_LOG.debug "Components started"
+
+        @queue.pop #<-------- stops here until interrupted
+
+        @clients.each do |name, component|
+          EUSTON_LOG.debug "Stopping component: #{name}"
+          component.stop
+        end
+
+        report_shutdown_reasons
+        EUSTON_LOG.debug "Components stopped"
+      end
+    end
+  end
+end

data/lib/euston-daemons/message_buffer_daemon/lib/message_logger.rb

@@ -0,0 +1,54 @@
+module Euston
+  class MessageLogger
+    class << self
+      def commands_logger channel
+        self.new channel, :commands
+      end
+
+      def events_logger channel
+        self.new channel, :events
+      end
+    end
+
+    include Euston::RabbitMq::Exchanges
+    include Euston::RabbitMq::Queues
+
+    def initialize channel, exchange_name
+      @channel = channel
+      @exchange = get_exchange channel, exchange_name
+      @read_model = Euston::MessageBufferDaemon::ReadModel::MessageLog.send exchange_name
+
+      @queue = get_queue channel, "#{exchange_name}_log"
+      @queue.bind @exchange, :routing_key => "#{exchange_name}.#"
+
+      @queue.when(:message_decode_failed => method(:log_failure),
+                  :message_failed        => method(:message_failed),
+                  :message_received      => method(:write_message_to_log))
+    end
+
+    def message_failed(message, error, header)
+      log_failure message, error
+      header.ack
+    end
+
+    def disconnect
+      @channel.disconnect
+    end
+
+    def start
+      @queue.safe_subscribe
+    end
+
+    def log_failure message, error
+      text = "A log queue subscription failed. [Error] #{error.message} [Payload] #{message}"
+      err = Euston::RabbitMq::MessageDecodeFailedError.new text
+      err.set_backtrace error.backtrace
+
+      Safely.report! err
+    end
+
+    def write_message_to_log message
+      @read_model.log_new_message message
+    end
+  end
+end

data/lib/euston-daemons/message_buffer_daemon/lib/publisher.rb

@@ -0,0 +1,56 @@
+module Euston
+  module MessageBufferDaemon
+    class Publisher
+      class << self
+        def commands_buffer channel
+          self.new channel, :commands
+        end
+
+        def events_buffer channel
+          self.new channel, :events
+        end
+      end
+
+      include Euston::RabbitMq::Exchanges
+
+      def initialize channel, exchange_name
+        @read_model = Euston::MessageBufferDaemon::ReadModel::MessageBuffer.send exchange_name
+
+        @channel = channel
+        @exchange = get_exchange @channel, exchange_name
+      end
+
+      def disconnect
+        @channel.disconnect
+      end
+
+      def dispatch_one_due_message
+        if (message = @read_model.find_next_message)
+          @read_model.set_next_attempt message
+          protected_publish(message)
+        end
+      end
+
+      def dispatch_due_messages
+        dispatched = 0
+        @read_model.find_due_messages.each do |message|
+          @read_model.set_next_attempt message
+          break unless protected_publish(message)
+          dispatched += 1
+        end
+        dispatched
+      end
+
+      def protected_publish(message)
+        ret = true
+        begin
+          @exchange.publish message['json'], default_publish_options.merge(:routing_key => "#{@exchange.name}.#{message['type']}")
+        rescue NativeException => e
+          Thread.current[:exception] = e
+          ret = false
+        end
+        ret
+      end
+    end
+  end
+end

data/lib/euston-daemons/message_buffer_daemon/lib/read_model/message_buffer.rb

@@ -0,0 +1,54 @@
+module Euston
+  module MessageBufferDaemon
+    module ReadModel
+      class MessageBuffer
+        class << self
+          def commands
+            self.new "buffered_commands", Euston::RabbitMq.read_model_mongodb
+          end
+
+          def events
+            self.new "buffered_events", Euston::RabbitMq.event_store_mongodb
+          end
+        end
+
+        def initialize name, mongodb
+          mongodb.create_collection name unless mongodb.collection_names.include? name
+
+          @collection = mongodb.collection name
+          @collection.ensure_index [ ['next_attempt', Mongo::ASCENDING] ], :unique => false, :name => "#{name}_next_attempt_index"
+        end
+
+        def buffer_new_message message
+          @collection.save({ 'doc_id' => message[:headers][:id],
+                             'type' => message[:headers][:type],
+                             'next_attempt' => (Time.now.to_f - 1.0),
+                             'json' => ActiveSupport::JSON.encode(message) }, :safe => { :fsync => true })
+        end
+
+        def find_next_message
+          query = { 'next_attempt' => { '$lte' => Time.now.to_f } }
+          @collection.find_one(query)
+        end
+
+        def find_due_messages
+          query = { 'next_attempt' => { '$lte' => Time.now.to_f }}
+          @collection.find(query)
+        end
+
+        def get_by_id id
+          @collection.find_one({ 'doc_id' => id }) #doc or nil
+        end
+
+        def set_next_attempt message
+          @collection.update({ 'doc_id' => message['doc_id'] },
+                             { '$set' => { 'next_attempt' => Time.now.to_f + 10 } }, :safe => { :fsync => true })
+        end
+
+        def remove_published_message id
+          @collection.remove({ 'doc_id' => id }, :safe => { :fsync => true })
+        end
+      end
+    end
+  end
+end

data/lib/euston-daemons/message_buffer_daemon/lib/read_model/message_log.rb

@@ -0,0 +1,36 @@
+module Euston
+  module MessageBufferDaemon
+    module ReadModel
+      class MessageLog
+        class << self
+          def commands
+            self.new "command_log", Euston::RabbitMq.read_model_mongodb
+          end
+
+          def events
+            self.new "event_log", Euston::RabbitMq.event_store_mongodb
+          end
+        end
+
+        def initialize name, mongodb
+          mongodb.create_collection name unless mongodb.collection_names.include? name
+
+          @collection = mongodb.collection name
+          @collection.ensure_index [ ['timestamp', Mongo::ASCENDING] ], :unique => false, :name => "#{name}_timestamp_index"
+        end
+
+        def find_all
+          @collection.find({}, { :sort => [ 'timestamp', Mongo::DESCENDING ] })
+        end
+
+        def log_new_message message
+          @collection.save({ '_id' => message[:headers][:id],
+                             'type' => message[:headers][:type],
+                             'version' => message[:headers][:version],
+                             'timestamp' => Time.now.to_f,
+                             'json' => ActiveSupport::JSON.encode(message) }, :safe => { :fsync => true })
+        end
+      end
+    end
+  end
+end

data/lib/euston-daemons/message_buffer_daemon/lib/settings.rb

@@ -0,0 +1,14 @@
+module Euston
+  module MessageBufferDaemon
+    module Settings
+      def self.configure(cfg=nil)
+        @config ||= {}
+        @config.merge!(cfg) if cfg && cfg.is_a?(Hash)
+      end
+
+      def self.debug
+        @config[:debug]
+      end
+    end
+  end
+end