euston-daemons 1.0.5 → 1.2.1

data/lib/euston-daemons/pipeline/config/environment.rb

@@ -0,0 +1,78 @@
+require 'i18n'
+require 'hot_bunnies'
+require 'macaddr'
+require 'euston-rabbitmq'
+
+require 'euston-daemons/pipeline/lib/command_processor/failed_message'
+require 'euston-daemons/pipeline/lib/command_processor/default_handlers/retry_failed_message'
+require 'euston-daemons/pipeline/lib/command_processor/component'
+require 'euston-daemons/pipeline/lib/command_logger/log'
+require 'euston-daemons/pipeline/lib/command_logger/component'
+require 'euston-daemons/pipeline/lib/event_processor/default_handlers/message_failure'
+require 'euston-daemons/pipeline/lib/event_processor/component'
+require 'euston-daemons/pipeline/lib/event_store_dispatcher/component'
+require 'euston-daemons/pipeline/lib/message_buffer/buffer'
+require 'euston-daemons/pipeline/lib/message_buffer/component'
+require 'euston-daemons/pipeline/lib/snapshotter/component'
+require 'euston-daemons/pipeline/lib/daemon'
+
+module Euston
+  module Daemons
+    module Pipeline
+      class DaemonEnvironment < Euston::DaemonEnvironment
+        class << self
+          attr_accessor :command_processors,
+                        :command_handler_namespaces,
+                        :command_loggers,
+                        :command_log_ignores,
+                        :event_handler_namespaces,
+                        :event_processors,
+                        :event_store_dispatchers,
+                        :event_store_dispatcher_wait_time,
+                        :message_buffers,
+                        :message_buffer_wait_time,
+                        :snapshotters,
+                        :snapshotter_wait_time,
+                        :snapshot_threshold,
+                        :user_defined_components
+        end
+
+        def initialize data
+          @logger             = data[:logger]
+          @daemon_config      = data[:daemon_config]
+          @i18n_locales_path  = @daemon_config[:i18n_locales_path]
+
+          @amqp_config  = ErbYaml.read data[:amqp_config_path],   data[:environment]
+          @mongo_config = ErbYaml.read data[:mongo_config_path],  data[:environment]
+
+          self.class.command_handler_namespaces       = data[:command_handler_namespaces]
+          self.class.event_handler_namespaces         = data[:event_handler_namespaces]
+          self.class.user_defined_components          = data[:user_defined_components]
+
+          self.class.command_processors               = @daemon_config[:command_processors].to_i
+          self.class.command_loggers                  = @daemon_config[:command_loggers].to_i
+          self.class.command_log_ignores              = @daemon_config[:command_log_ignores]
+          self.class.event_processors                 = @daemon_config[:event_processors].to_i
+          self.class.event_store_dispatchers          = @daemon_config[:event_store_dispatchers].to_i
+          self.class.event_store_dispatcher_wait_time = @daemon_config[:event_store_dispatcher_wait_time].to_f
+          self.class.message_buffers                  = @daemon_config[:message_buffers].to_i
+          self.class.message_buffer_wait_time         = @daemon_config[:message_buffer_wait_time].to_f
+          self.class.snapshotters                     = @daemon_config[:snapshotters].to_i
+          self.class.snapshotter_wait_time            = @daemon_config[:snapshotter_wait_time].to_f
+          self.class.snapshot_threshold               = @daemon_config[:snapshot_threshold].to_i
+        end
+
+        def setup
+          setup_safely @daemon_config[:hoptoad_key]
+          setup_amqp @amqp_config
+          setup_mongo @mongo_config
+          setup_euston @mongo_config[:event_store_database]
+
+          I18n.load_path += Dir[@i18n_locales_path] unless @i18n_locales_path.nil?
+
+          self.class
+        end
+      end
+    end
+  end
+end

data/lib/euston-daemons/pipeline/lib/command_logger/component.rb

@@ -0,0 +1,54 @@
+module Euston
+  module Daemons
+    module Pipeline
+      module CommandLogger
+        class Component < Euston::DaemonComponent
+          extend RabbitMq::Exchanges
+          extend RabbitMq::Queues
+
+          def initialize channel, id = 1, logger = Euston::NullLogger.instance
+            @id = id
+            @log = logger
+            @channel = channel
+            @channel.prefetch = 1
+            @command_log = Log.new DaemonEnvironment.event_store_mongodb
+            @exchange = self.class.get_exchange channel, :commands
+            @queue = self.class.get_queue channel, @command_log.name
+            @queue.bind @exchange, :routing_key => "#{@exchange.name}.#"
+
+            @queue.when(:message_decode_failed => method(:log_failure),
+                        :message_failed        => method(:message_failed),
+                        :message_received      => method(:write_message_to_log))
+
+            @consumer = @queue.consumer
+          end
+
+          private
+
+          def log_failure message, error
+            return if ignorable_exception? error
+
+            text = "Command logger #{@id} failed.\n    [Error] #{error.message}\n    [Payload] #{message}"
+            err = Euston::RabbitMq::MessageDecodeFailedError.new text
+            err.set_backtrace error.backtrace
+
+            Safely.report! err
+          end
+
+          def message_failed message, error, reactive_message
+            log_failure message, error
+            reactive_message.ack
+          end
+
+          def next_iteration
+            @queue.safe_subscribe_with_timeout @consumer
+          end
+
+          def write_message_to_log message
+            @command_log.write_command message unless DaemonEnvironment.command_log_ignores.include? message[:headers][:type]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/euston-daemons/pipeline/lib/command_logger/log.rb

@@ -0,0 +1,31 @@
+module Euston
+  module Daemons
+    module Pipeline
+      module CommandLogger
+        class Log
+          def initialize mongodb
+            @name = 'command_log'
+            mongodb.create_collection @name unless mongodb.collection_names.include? @name
+
+            @collection = mongodb.collection @name
+          end
+
+          attr_reader :name
+
+          def write_command command
+            timestamp = Time.now.to_f
+
+            doc = { '_id'                   => command[:headers][:id],
+                    'type'                  => command[:headers][:type],
+                    'version'               => command[:headers][:version],
+                    'timestamp'             => timestamp,
+                    'timestamp_for_humans'  => Time.at(timestamp),
+                    'json'                  => ActiveSupport::JSON.encode(command) }
+
+            @collection.insert doc
+          end
+        end
+      end
+    end
+  end
+end

data/lib/euston-daemons/pipeline/lib/command_processor/component.rb

@@ -0,0 +1,50 @@
+module Euston
+  module Daemons
+    module Pipeline
+      module CommandProcessor
+        class Component < Euston::DaemonComponent
+          include Euston::CommandHandlerPrivateMethodNames
+
+          def initialize channel, handlers, id = 1, logger = Euston::NullLogger.instance
+            @channel = channel
+            @channel.prefetch = 1
+            @handlers = handlers
+            @id = id
+            @log = logger
+            @process_method = method(:process_message)
+            @stopwatch = Stopwatch.new.when(:finished => method(:log_elapsed_time))
+          end
+
+          private
+
+          def next_iteration
+            Euston::RabbitMq::RetryingSubscription.new(@channel, :command_handlers, @log)
+              .when(:message_received => @process_method)
+              .subscribe
+          end
+
+          def process_message message
+            @stopwatch.time do
+              command_headers = CommandHeaders.from_hash(message[:headers]).freeze
+              command_body = message[:body].freeze
+
+              handler_type = command_headers.type.to_s.camelize.to_sym
+              reference = @handlers.find { |reference| reference.name == handler_type }
+
+              if reference.nil?
+                Euston::CommandBus.publish command_headers, command_body, @log
+              else
+                handler_method_name = self.class.command_handler_method_name command_headers.version
+                reference.handler.new.send handler_method_name, command_headers, command_body
+              end
+            end
+          end
+
+          def log_elapsed_time elapsed_time
+            @log.debug "Command handler client #{@id} processed a message in #{elapsed_time} sec(s)"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/euston-daemons/pipeline/lib/command_processor/default_commands/retry_failed_message.rb

@@ -0,0 +1,13 @@
+module Euston
+  module Daemons
+    module Pipeline
+      module CommandProcessor
+        module DefaultCommands
+          class RetryFailedMessage < Euston::Command
+            validates :id, :presence => true, :format => { :with => /^([0-9a-fA-F]){8}-([0-9a-fA-F]){4}-([0-9a-fA-F]){4}-([0-9a-fA-F]){4}-([0-9a-fA-F]){12}$/ }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/euston-daemons/pipeline/lib/command_processor/default_handlers/retry_failed_message.rb

@@ -0,0 +1,34 @@
+module Euston
+  module Daemons
+    module Pipeline
+      module CommandProcessor
+        module DefaultHandlers
+          class RetryFailedMessage
+            include Euston::CommandHandler
+
+            version 1 do |headers, command|
+              failed_messages = FailedMessage.new DaemonEnvironment.event_store_mongodb
+              message = failed_messages.get_by_id command[:id]
+
+              unless message.nil?
+                routing_key = message['routing_key']
+                exchange = routing_key.split('.').first
+
+                headers = message['headers'].merge(:id => command[:id],
+                                                   :type => message['type'],
+                                                   :version => message['version'] )
+
+                retry_message = { :headers => headers, :body => message['body'] }
+
+                buffer = MessageBuffer::Buffer.new DaemonEnvironment.event_store_mongodb
+                buffer.enqueue exchange, retry_message
+
+                failed_messages.remove_by_id command[:id]
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/euston-daemons/pipeline/lib/command_processor/failed_message.rb

@@ -0,0 +1,36 @@
+module Euston
+  module Daemons
+    module Pipeline
+      module CommandProcessor
+        class FailedMessage
+          def initialize mongodb
+            name = 'failed_messages'
+            mongodb.create_collection name unless mongodb.collection_names.include? name
+
+            @collection = mongodb.collection name
+            @collection.ensure_index [ ['failure_timestamp', Mongo::ASCENDING] ], :unique => false, :name => 'failed_messages_failure_timestamp_index'
+          end
+
+          def get_by_id id
+            @collection.find_one({ '_id' => id })
+          end
+
+          def find_all
+            @collection.find({}, { :sort => [ 'failure_timestamp', Mongo::DESCENDING ] })
+          end
+
+          def log_failure failure
+            failure.recursive_stringify_keys!
+            failure['_id'] = failure.delete 'message_id'
+
+            @collection.save(failure, :safe => { :fsync => true })
+          end
+
+          def remove_by_id id
+            @collection.remove({ '_id' => id }, :safe => { :fsync => true })
+          end
+        end
+      end
+    end
+  end
+end

data/lib/euston-daemons/pipeline/lib/daemon.rb

@@ -0,0 +1,85 @@
+module Euston
+  module Daemons
+    module Pipeline
+      class Daemon < Euston::Daemon
+        private
+
+        def env
+          DaemonEnvironment
+        end
+
+        def post_shutdown_cleanup
+          env.amqp_connection.close
+        end
+
+        def pre_registration_setup
+          channel = env.amqp_connection.create_channel
+
+          begin
+            handler_finder = RabbitMq::HandlerFinder.new [AggregateRoot, Euston::CommandHandler], @log
+            handler_finder.namespaces.push Pipeline::CommandProcessor::DefaultHandlers, *DaemonEnvironment.command_handler_namespaces
+            @command_handler_references = handler_finder.find
+
+            binder = RabbitMq::CommandHandlerBinder.new channel, @command_handler_references, @log
+            binder.ensure_bindings_exist
+
+            handler_finder = RabbitMq::HandlerFinder.new [Euston::EventHandler], @log
+            handler_finder.namespaces.push Pipeline::EventProcessor::DefaultHandlers, *DaemonEnvironment.event_handler_namespaces
+            @event_handler_references = handler_finder.find
+
+            binder = RabbitMq::EventHandlerBinder.new channel, @event_handler_references, @log
+            binder.ensure_bindings_exist
+          rescue NativeException => e
+            @log.error "Caught the following native exception:\n\n#{e}\n\nDid you forget to create your rabbitmq vhost and user for this environment?"
+            raise e
+          ensure
+            channel.close
+          end
+        end
+
+        def register_components
+          (1..env.command_loggers).each do |i|
+            component = CommandLogger::Component.new env.amqp_connection.create_channel, i, @log
+            register_component "command_logger_#{i}".to_sym, component
+          end
+
+          (1..env.command_processors).each do |i|
+            component = CommandProcessor::Component.new env.amqp_connection.create_channel, @command_handler_references, i, @log
+            register_component "command_processor_#{i}", component
+          end
+
+          (1..env.event_store_dispatchers).each do |i|
+            component = EventStoreDispatcher::Component.new env.amqp_connection.create_channel, "#{Mac.addr.to_s} #{i}", @log
+            component = register_component "event_store_dispatcher_#{i}", component
+            component.wait_time = env.event_store_dispatcher_wait_time
+          end
+
+          (1..env.message_buffers).each do |i|
+            component = MessageBuffer::Component.new env.amqp_connection.create_channel, "#{Mac.addr.to_s} #{i}", @log
+            component = register_component "message_buffer_#{i}", component
+            component.wait_time = env.message_buffer_wait_time
+          end
+
+          (1..env.snapshotters).each do |i|
+            component = Snapshotter::Component.new Euston::Repository.event_store, env.snapshot_threshold, i, @log
+            component = register_component "snapshotter_#{i}", component
+            component.wait_time = env.snapshotter_wait_time
+          end
+
+          (1..env.event_processors).each do |i|
+            component = EventProcessor::Component.new env.amqp_connection.create_channel, @event_handler_references, i, @log
+            register_component "event_processor_#{i}", component
+          end
+
+          env.user_defined_components.each do |udc|
+            udc[:quantity].times do |i|
+              component = udc[:factory_method].call env, i, @log
+              component = register_component "#{udc[:name]}_#{i}", component
+              component.wait_time = udc[:wait_time] || 0.2
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/euston-daemons/pipeline/lib/event_processor/component.rb

@@ -0,0 +1,67 @@
+module Euston
+  module Daemons
+    module Pipeline
+      module EventProcessor
+        class Component < Euston::DaemonComponent
+          include Euston::EventHandlerPrivateMethodNames
+
+          def initialize channel, references, id = 1, logger = Euston::NullLogger.instance
+            @channel = channel
+            @channel.prefetch = 1
+            @id = id
+            @references = references
+            @log = logger
+            @process_method = method(:process_message)
+            @stopwatch = Stopwatch.new.when(:finished => method(:log_elapsed_time))
+
+            @subscriptions = @references.map do |r|
+              { :reference => r,
+                :subscription => RabbitMq::RetryingSubscription.new(@channel, r.name.to_s.underscore, @log)
+                                                               .when(:message_received => @process_method) }
+            end
+          end
+
+          private
+
+          def next_iteration
+            @subscriptions.each do |s|
+              @reference = s[:reference]
+              s[:subscription].get
+            end
+          end
+
+          def process_message message
+            @stopwatch.time do
+              event_headers = EventHeaders.from_hash message[:headers]
+              event_body = message[:body]
+
+              handler_is_aggregate_root = @reference.handler.include? Euston::AggregateRoot
+
+              if handler_is_aggregate_root
+                id_getter_method = self.class.id_from_event_method_name event_headers.type, event_headers.version
+                id = @reference.handler.send id_getter_method, event_body
+
+                raise "Unable to extract the aggregate root id from an #{event_headers.type} event. Did you forget to add an { :id => :field_name } argument to your subscribes block?" if id.nil?
+
+                handler_instance = Euston::Repository.find @reference.handler, id
+                handler_instance.log = @log
+                handler_instance.consume_event_subscription event_headers.freeze, event_body.freeze
+
+                Euston::Repository.save handler_instance
+              else
+                handler_method = self.class.event_handler_method_name event_headers.type, event_headers.version
+                handler_instance = @reference.handler.new
+                handler_instance.log = @log if handler_instance.respond_to? :log=
+                handler_instance.send handler_method, event_headers.freeze, event_body.freeze
+              end
+            end
+          end
+
+          def log_elapsed_time elapsed_time
+            @log.debug "Event handler client #{@reference.name.to_s.underscore} processed a message in #{elapsed_time} sec(s)"
+          end
+        end
+      end
+    end
+  end
+end