euston-daemons 1.1.0-java → 1.2.0-java

data/lib/euston-daemons/pipeline/lib/daemon.rb

@@ -0,0 +1,77 @@
+module Euston
+  module Daemons
+    module Pipeline
+      class Daemon < Euston::Daemon
+        private
+
+        def env
+          DaemonEnvironment
+        end
+
+        def post_shutdown_cleanup
+          env.amqp_connection.close
+        end
+
+        def pre_registration_setup
+          channel = env.amqp_connection.create_channel
+
+          begin
+            handler_finder = RabbitMq::HandlerFinder.new [AggregateRoot, Euston::CommandHandler], @log
+            handler_finder.namespaces.push Pipeline::CommandProcessor::DefaultHandlers, *DaemonEnvironment.command_handler_namespaces
+            @command_handler_references = handler_finder.find
+
+            binder = RabbitMq::CommandHandlerBinder.new channel, @command_handler_references, @log
+            binder.ensure_bindings_exist
+
+            handler_finder = RabbitMq::HandlerFinder.new [Euston::EventHandler], @log
+            handler_finder.namespaces.push Pipeline::EventProcessor::DefaultHandlers, *DaemonEnvironment.event_handler_namespaces
+            @event_handler_references = handler_finder.find
+
+            binder = RabbitMq::EventHandlerBinder.new channel, @event_handler_references, @log
+            binder.ensure_bindings_exist
+          rescue NativeException => e
+            @log.error "Caught the following native exception:\n\n#{e}\n\nDid you forget to create your rabbitmq vhost and user for this environment?"
+            raise e
+          ensure
+            channel.close
+          end
+        end
+
+        def register_components
+          (1..env.command_loggers).each do |i|
+            component = CommandLogger::Component.new env.amqp_connection.create_channel, i, @log
+            register_component "command_logger_#{i}".to_sym, component
+          end
+
+          (1..env.command_processors).each do |i|
+            component = CommandProcessor::Component.new env.amqp_connection.create_channel, @command_handler_references, i, @log
+            register_component "command_processor_#{i}".to_sym, component
+          end
+
+          (1..env.event_store_dispatchers).each do |i|
+            component = EventStoreDispatcher::Component.new env.amqp_connection.create_channel, "#{Mac.addr.to_s} #{i}", @log
+            component = register_component "event_store_dispatcher_#{i}", component
+            component.wait_time = env.event_store_dispatcher_wait_time
+          end
+
+          (1..env.message_buffers).each do |i|
+            component = MessageBuffer::Component.new env.amqp_connection.create_channel, "#{Mac.addr.to_s} #{i}", @log
+            component = register_component "message_buffer_#{i}", component
+            component.wait_time = env.message_buffer_wait_time
+          end
+
+          (1..env.snapshotters).each do |i|
+            component = Snapshotter::Component.new Euston::Repository.event_store, env.snapshot_threshold, i, @log
+            component = register_component "snapshotter_#{i}", component
+            component.wait_time = env.snapshotter_wait_time
+          end
+
+          (1..env.event_processors).each do |i|
+            component = EventProcessor::Component.new env.amqp_connection.create_channel, @event_handler_references, i, @log
+            register_component "event_processor_#{i}".to_sym, component
+          end
+        end
+      end
+    end
+  end
+end

data/lib/euston-daemons/pipeline/lib/event_processor/component.rb

@@ -0,0 +1,67 @@
+module Euston
+  module Daemons
+    module Pipeline
+      module EventProcessor
+        class Component < Euston::DaemonComponent
+          include Euston::EventHandlerPrivateMethodNames
+
+          def initialize channel, references, id = 1, logger = Euston::NullLogger.instance
+            @channel = channel
+            @channel.prefetch = 1
+            @id = id
+            @references = references
+            @log = logger
+            @process_method = method(:process_message)
+            @stopwatch = Stopwatch.new.when(:finished => method(:log_elapsed_time))
+
+            @subscriptions = @references.map do |r|
+              { :reference => r,
+                :subscription => RabbitMq::RetryingSubscription.new(@channel, r.name.to_s.underscore, @log)
+                                                               .when(:message_received => @process_method) }
+            end
+          end
+
+          private
+
+          def next_iteration
+            @subscriptions.each do |s|
+              @reference = s[:reference]
+              s[:subscription].get
+            end
+          end
+
+          def process_message message
+            @stopwatch.time do
+              event_headers = EventHeaders.from_hash message[:headers]
+              event_body = message[:body]
+
+              handler_is_aggregate_root = @reference.handler.include? Euston::AggregateRoot
+
+              if handler_is_aggregate_root
+                id_getter_method = self.class.id_from_event_method_name event_headers.type, event_headers.version
+                id = @reference.handler.send id_getter_method, event_body
+
+                raise "Unable to extract the aggregate root id from an #{event_headers.type} event. Did you forget to add an { :id => :field_name } argument to your subscribes block?" if id.nil?
+
+                handler_instance = Euston::Repository.find @reference.handler, id
+                handler_instance.log = @log
+                handler_instance.consume_event_subscription event_headers.freeze, event_body.freeze
+
+                Euston::Repository.save handler_instance
+              else
+                handler_method = self.class.event_handler_method_name event_headers.type, event_headers.version
+                handler_instance = @reference.handler.new
+                handler_instance.log = @log if handler_instance.respond_to? :log=
+                handler_instance.send handler_method, event_headers.freeze, event_body.freeze
+              end
+            end
+          end
+
+          def log_elapsed_time elapsed_time
+            @log.debug "Event handler client #{@reference.name.to_s.underscore} processed a message in #{elapsed_time} sec(s)"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/euston-daemons/pipeline/lib/event_processor/default_handlers/message_failure.rb

@@ -0,0 +1,30 @@
+module Euston
+  module Daemons
+    module Pipeline
+      module EventProcessor
+        module DefaultHandlers
+          class MessageFailure
+            include Euston::EventHandler
+
+            subscribes :message_failed, 1 do |headers, event|
+              headers = event[:message][:headers].dup
+              failure = { :message_id         => headers.delete(:id),
+                          :type               => headers.delete(:type),
+                          :version            => headers.delete(:version),
+                          :message_timestamp  => headers.delete(:timestamp),
+                          :routing_key        => event[:routing_key],
+                          :body               => event[:message][:body],
+                          :headers            => headers,
+                          :error              => event[:error],
+                          :backtrace          => event[:backtrace],
+                          :failure_timestamp  => Time.now.to_f }
+
+              failed_messages = CommandProcessor::FailedMessage.new DaemonEnvironment.event_store_mongodb
+              failed_messages.log_failure failure
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/euston-daemons/pipeline/lib/event_store_dispatcher/component.rb

@@ -0,0 +1,68 @@
+module Euston
+  module Daemons
+    module Pipeline
+      module EventStoreDispatcher
+        class Component < DaemonComponent
+          extend RabbitMq::Exchanges
+
+          def initialize channel, id = 1, log = Euston::NullLogger.instance
+            @channel = channel
+            @channel.tx_select
+            @id = id
+            @log = log
+            @event_store = DaemonEnvironment.event_store
+            @stopwatch = Stopwatch.new.when(:finished => method(:log_elapsed_time))
+            @commands_exchange = self.class.get_exchange channel, :commands
+            @events_exchange = self.class.get_exchange channel, :events
+            @buffer = MessageBuffer::Buffer.new DaemonEnvironment.event_store_mongodb
+          end
+
+          private
+
+          def log_elapsed_time elapsed_time
+            @log.debug "Event store dispatcher #{@id} dispatched #{@commits_dispatched} commit(s) in #{elapsed_time} sec(s)" if @commits_dispatched > 0
+          end
+
+          def next_iteration
+            @commits_dispatched = 0
+
+            @stopwatch.time do
+              begin
+                @event_store.take_ownership_of_undispatched_commits @id
+                @commits = @event_store.get_undispatched_commits @id
+
+                begin
+                  @commits.each do |commit|
+                    commit.events.each do |event|
+                      hash = event.to_hash
+                      @events_exchange.publish ActiveSupport::JSON.encode(hash), self.class.default_publish_options.merge(:routing_key => "events.#{hash[:headers][:type]}")
+                    end
+
+                    commit.commands.each do |command|
+                      hash = command.to_hash
+
+                      if hash[:headers][:dispatch_at].nil?
+                        @commands_exchange.publish ActiveSupport::JSON.encode(hash), self.class.default_publish_options.merge(:routing_key => "commands.#{hash[:headers][:type]}")
+                      else
+                        @buffer.enqueue :commands, hash, hash[:headers][:dispatch_at]
+                      end
+                    end
+                  end
+
+                  @channel.tx_commit
+                rescue StandardError => e
+                  @channel.tx_rollback
+                  raise e
+                end
+
+                @event_store.mark_commits_as_dispatched @commits
+
+                @commits_dispatched += @commits.size
+              end until stopped || @commits.empty?
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/euston-daemons/pipeline/lib/message_buffer/buffer.rb

@@ -0,0 +1,85 @@
+module Euston
+  module Daemons
+    module Pipeline
+      module MessageBuffer
+        class Buffer
+          def initialize mongodb
+            name = 'message_buffer'
+            mongodb.create_collection name unless mongodb.collection_names.include? name
+
+            @name = name
+            @collection = mongodb.collection name
+
+            @collection.ensure_index [ ['message_id', Mongo::ASCENDING] ],
+                                       :unique  => false,
+                                       :name    => "#{name}_message_id_index"
+
+            @collection.ensure_index [ ['component_id', Mongo::ASCENDING],
+                                       ['dispatch_at',  Mongo::ASCENDING] ],
+                                       :unique  => false,
+                                       :name    => "#{name}_component_id_dispatch_at_index"
+          end
+
+          attr_reader :name
+
+          def delete_dispatched_messages component_id
+            @collection.remove({ 'component_id' => component_id }, :multi => true)
+          end
+
+          def enqueue exchange, message, dispatch_at = nil
+            messages = message
+            messages = [{ :hash => message, :dispatch_at => dispatch_at }] unless messages.is_a? Array
+
+            messages = messages.map do |m|
+              message_is_well_formed = m.is_a?(Hash) && m.has_key?(:hash) && m.has_key?(:dispatch_at)
+              m = { :hash => m, :dispatch_at => dispatch_at } unless message_is_well_formed
+              map_to_document exchange, m
+            end
+
+            @collection.insert(messages) unless messages.empty?
+          end
+
+          def find_dispatchable_messages component_id
+            query   = { 'component_id' => component_id }
+            fields  = ['exchange', 'type', 'json']
+            sort    = [ 'dispatch_at', Mongo::ASCENDING ]
+
+            @collection.find query, :fields => fields, :sort => sort
+          end
+
+          def get_by_id id
+            @collection.find_one 'message_id' => id
+          end
+
+          def take_ownership_of_dispatchable_messages component_id
+            new_messages_eligible_for_dispatch  = { 'component_id'  => '',
+                                                    'dispatch_at'   => { '$lte' => Time.now.to_f } }
+
+            messages_stuck_in_other_components  = { 'component_id'  => { '$ne'  => '' },
+                                                    'dispatch_at'   => { '$lte' => Time.now.to_f - 60 } }
+
+            query = { '$or'   => [ new_messages_eligible_for_dispatch, messages_stuck_in_other_components ] }
+            doc   = { '$set'  => { 'component_id' => component_id } }
+
+            @collection.update query, doc, :multi => true
+          end
+
+          private
+
+          def map_to_document exchange, message
+            hash        = message[:hash]
+            dispatch_at = (message[:dispatch_at] || Time.now.to_f).to_f
+
+            { 'message_id'              => hash[:headers][:id],
+              'exchange'                => exchange,
+              'type'                    => hash[:headers][:type],
+              'component_id'            => '',
+              'dispatch_at'             => dispatch_at,
+              'dispatch_at_for_humans'  => Time.at(dispatch_at),
+              'json'                    => ActiveSupport::JSON.encode(hash) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/euston-daemons/pipeline/lib/message_buffer/component.rb

@@ -0,0 +1,59 @@
+module Euston
+  module Daemons
+    module Pipeline
+      module MessageBuffer
+        class Component < Euston::DaemonComponent
+          extend RabbitMq::Exchanges
+
+          def initialize channel, id = 1, logger = Euston::NullLogger.instance
+            @channel = channel
+            @channel.tx_select
+            @id = "message_buffer #{id}"
+            @log = logger
+            @buffer = Buffer.new DaemonEnvironment.event_store_mongodb
+            @stopwatch = Stopwatch.new.when(:finished => method(:log_elapsed_time))
+          end
+
+          private
+
+          def dispatch_due_messages
+            @dispatched_count = 0
+
+            @stopwatch.time do
+              @buffer.take_ownership_of_dispatchable_messages @id
+
+              begin
+                @buffer.find_dispatchable_messages(@id).each do |message|
+                  exchange = self.class.get_exchange @channel, message['exchange']
+                  exchange.publish message['json'], self.class.default_publish_options.merge(:routing_key => "#{exchange.name}.#{message['type']}")
+                  @dispatched_count += 1
+                end
+
+                @channel.tx_commit
+              rescue StandardError => e
+                @channel.tx_rollback
+                raise e
+              end
+
+              @buffer.delete_dispatched_messages @id
+            end
+
+            @dispatched_count
+          end
+
+          def log_elapsed_time elapsed_time
+            @log.debug "Message buffer #{@id} dispatched #{@dispatched_count} message(s) in #{elapsed_time} sec(s)" if @dispatched_count > 0
+          end
+
+          def next_iteration
+            @messages_dispatched = 0
+
+            begin
+              @messages_dispatched = dispatch_due_messages
+            end until stopped || @messages_dispatched.zero?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/euston-daemons/pipeline/lib/snapshotter/component.rb

@@ -0,0 +1,48 @@
+module Euston
+  module Daemons
+    module Pipeline
+      module Snapshotter
+        class Component < Euston::DaemonComponent
+          def initialize event_store, threshold, id = 1, logger = Euston::NullLogger.instance
+            @event_store = event_store
+            @threshold = threshold
+            @id = id
+            @log = logger
+          end
+
+          private
+
+          def next_iteration
+            begin
+              stream_heads = @event_store.get_streams_to_snapshot @threshold
+              @log.debug "Found #{stream_heads.length} stream(s) eligible for snapshotting (threshold is #{@threshold})" if stream_heads.any?
+
+              stream_heads.each do |stream_head|
+                pair = @event_store.get_snapshot_stream_pair stream_head.stream_id
+
+                loader = RabbitMq::ConstantLoader.new
+                loader.when(:hit => ->(klass) { take_snapshot klass, pair },
+                            :miss => ->(type) { Safely.report! "Snapshotter was unable to find a class: #{type}" })
+
+                loader.load pair.stream.committed_headers[:aggregate_type]
+              end
+            end until stopped || stream_heads.empty?
+          end
+
+          def take_snapshot klass, pair
+            instance = klass.hydrate pair.stream, pair.snapshot
+            snapshot = instance.take_snapshot
+            snapshot = EventStore::Snapshot.new pair.stream.stream_id,
+                                                pair.stream.stream_revision,
+                                                snapshot[:payload],
+                                                :version => snapshot[:version]
+
+            @log.debug "Writing snapshot: #{snapshot.inspect}"
+
+            @event_store.add_snapshot snapshot
+          end
+        end
+      end
+    end
+  end
+end

data/lib/euston-daemons/pipeline/rake_task.rb

@@ -0,0 +1,45 @@
+module Euston
+  class PipelineRakeTask < Euston::Daemons::RakeTask
+    attr_accessor :amqp_config_path,
+                  :command_handler_namespaces,
+                  :daemon_config_path,
+                  :event_handler_namespaces,
+                  :mongo_config_path
+
+    def initialize environment
+      super(environment, :pipeline_daemon)
+    end
+
+    def before_creating_task
+      @daemon_path = File.expand_path(File.dirname __FILE__) + File::SEPARATOR
+      @daemon_class = 'Euston::Daemons::Pipeline::Daemon'
+    end
+
+    def load_daemon_config_file
+      @data[:daemon_config] = ErbYaml.read @daemon_config_path, @data[:environment]
+    end
+
+    def initialize_settings
+      @logger.debug "AMQP config path: #{@amqp_config_path}"
+      @data[:amqp_config_path] = @amqp_config_path
+
+      @logger.debug "Command handler namespaces: #{@command_handler_namespaces}"
+      @data[:command_handler_namespaces] = @command_handler_namespaces
+
+      @logger.debug "Daemon config path: #{@daemon_config_path}"
+
+      @logger.debug "Event handler namespaces: #{@event_handler_namespaces}"
+      @data[:event_handler_namespaces] = @event_handler_namespaces
+
+      @logger.debug "Mongo config path: #{@mongo_config_path}"
+      @data[:mongo_config_path] = @mongo_config_path
+    end
+
+    def load_environment
+      @logger.debug "Loading environment"
+      require 'euston-daemons/pipeline/config/environment'
+
+      Euston::Daemons::Pipeline::DaemonEnvironment.new(@data).setup
+    end
+  end
+end

data/lib/euston-daemons/rake_task.rb

@@ -15,11 +15,8 @@ module Euston
       # Daemon class. Must be supplied as a string.
       attr_accessor :daemon_class
 
-      # Logger instance. Defaults to writing to /var/log/euston.log.
-      attr_accessor :logger
-
-      # # Path to write the daemon pid file to
-      attr_accessor :pid_path
+      # Callable. Receives the daemon environment object to allow the user to perform other related config operations.
+      attr_accessor :post_setup_callback
 
       # Use verbose output. If this is set to true, the task will print the
       # executed command to stdout.
@@ -40,11 +37,13 @@ module Euston
         desc("Run a Euston daemon") unless ::Rake.application.last_comment
         task name do
           RakeFileUtils.send(:verbose, verbose) do
+            load_daemon_config_file
             initialize_logger
+            initialize_settings
             write_pid_file
             log_startup
-            initialize_settings
-            load_environment
+            env = load_environment
+            post_setup_callback.call env unless post_setup_callback.nil?
             launch_and_wait_for_exit
             log_shutdown
             remove_pid_file
@@ -54,16 +53,19 @@ module Euston
 
       private
 
-      def validate_environment environment
-        environment = environment.to_s.downcase.to_sym
-        environments = [:development, :test, :staging, :production]
-        environment = :development unless environments.include? environment
-        environment
-      end
-
       def initialize_logger
-        @logger ||= Logger.new(STDOUT)
-        @data[:logger] = @logger
+        config = @data[:daemon_config]
+        log_path = config[:log_path]
+
+        raise "Required log path does not exist: #{log_path}" unless Dir.exist? log_path
+
+        @data[:logger] = @logger = Logger.new(File.join log_path, "#{@name}.#{@environment}.log")
+
+        begin
+          @logger.level = Logger.const_get config[:log_level].upcase.to_sym
+        rescue
+          @logger.level = Logger::DEBUG
+        end
       end
 
       def initialize_settings
@@ -81,6 +83,14 @@ module Euston
         ns.new(@data).run
       end
 
+      def load_daemon_config_file
+        # virtual
+      end
+
+      def load_environment
+        # virtual
+      end
+
       def log_shutdown
         print_log_banner "Daemon shut down: #{@daemon_class}"
       end
@@ -105,16 +115,27 @@ module Euston
         File.delete @pid_file rescue Errno::ENOENT
       end
 
+      def validate_environment environment
+        environment = environment.to_s.downcase.to_sym
+        environments = [:development, :test, :staging, :production]
+        environment = :development unless environments.include? environment
+        environment
+      end
+
       def write_pid_file
-        @pid_file = File.join pid_path, "#{@name}.pid"
+        pid_path = @data[:daemon_config][:pid_path]
+
+        raise "Required pid path does not exist: #{pid_path}" unless Dir.exist? pid_path
+
+        @pid_file = File.join pid_path, "#{@name}.#{@environment}.pid"
 
         if defined? Java
-          @pid = Java::JavaIo::File.new("/proc/self").canonical_file.name
+          @pid = java.lang.management.ManagementFactory.getRuntimeMXBean().getName().split('@').first
         else
           @pid = File.readlink("/proc/self")
         end
 
-        @logger.debug "Writing pid #{@pid} to #{@pid_file}"
+        @logger.error "Writing pid #{@pid} to #{@pid_file}"
 
         File.open(@pid_file, 'w') { |f| f.puts @pid }
       end

data/lib/euston-daemons/rake_tasks.rb

@@ -2,6 +2,4 @@ require 'euston-daemons'
 
 require 'rake/tasklib'
 require 'euston-daemons/rake_task'
-require 'euston-daemons/command_processor_daemon/rake_task'
-require 'euston-daemons/event_processor_daemon/rake_task'
-require 'euston-daemons/message_buffer_daemon/rake_task'
+require 'euston-daemons/pipeline/rake_task'

data/lib/euston-daemons/version.rb

@@ -1,5 +1,5 @@
 module Euston
   module Daemons
-    VERSION = "1.1.0"
+    VERSION = "1.2.0"
   end
 end

data/lib/euston-daemons.rb

@@ -10,7 +10,9 @@ if RUBY_PLATFORM.to_s == 'java'
   end
 end
 
+require 'euston-daemons/euston/stopwatch'
+require 'euston-daemons/euston/exceptions'
 require 'euston-daemons/euston/daemon_environment'
-require 'euston-daemons/euston/daemon_client'
 require 'euston-daemons/euston/daemon_component'
+require 'euston-daemons/euston/daemon_component_host'
 require 'euston-daemons/euston/daemon'

data/spec/daemons/{command_handler_spec.rb → command_processor_spec.rb}

@@ -1,10 +1,10 @@
-describe 'command handler client', :purge_event_store, :purge_rabbitmq do
-  require 'euston-daemons/command_processor_daemon/lib/clients/command_handler'
+describe 'command handler component', :purge_event_store, :purge_rabbitmq do
+  require 'euston-daemons/pipeline/lib/command_processor/component'
 
   let(:logger)            { Euston::NullLogger.instance }
   let(:handlers)          { [] }
   let(:subscription)      { StubRetryingSubscription.new }
-  let(:client)            { Euston::CommandProcessorDaemon::CommandHandler.new @channel, handlers, logger }
+  let(:client)            { Euston::Daemons::Pipeline::CommandProcessor::Component.new @channel, handlers, 1, logger }
   let(:aggregate)         { Euston::Daemons::SampleModel::Counter2.new }
   let(:saved_aggregates)  { [] }
 

data/spec/daemons/{event_handler_spec.rb → event_processor_spec.rb}

@@ -1,9 +1,9 @@
 describe 'event handler client', :purge_event_store, :purge_rabbitmq do
-  require 'euston-daemons/event_processor_daemon/lib/clients/event_handler'
+  require 'euston-daemons/pipeline/lib/event_processor/component'
 
   let(:logger)            { Euston::NullLogger.instance }
   let(:subscription)      { StubRetryingSubscription.new }
-  let(:client)            { Euston::EventProcessorDaemon::EventHandler.new @channel, reference, logger }
+  let(:client)            { Euston::Daemons::Pipeline::EventProcessor::Component.new @channel, reference, 1, logger }
   let(:aggregate)         { Euston::Daemons::SampleModel::Counter2.new }
   let(:saved_aggregates)  { [] }