euston-daemons 1.1.0-java → 1.2.0-java

data/Gemfile

@@ -4,3 +4,4 @@ gemspec
 gem 'cranky',             :git => 'https://github.com/leemhenson/cranky.git',           :branch => 'hashes'
 gem 'ffaker',             :git => 'https://github.com/leemhenson/ffaker.git',           :branch => 'countries'
 gem 'safely',             :git => 'https://github.com/leemhenson/safely.git',           :branch => 'override_strategies'
+gem 'hot_bunnies',        :git => 'https://github.com/ruby-amqp/hot_bunnies.git'

data/euston-daemons.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name        = 'euston-daemons'
-  s.version     = '1.1.0'
+  s.version     = '1.2.0'
   s.platform    = RUBY_PLATFORM.to_s == 'java' ? 'java' : Gem::Platform::RUBY
   s.authors     = ['Lee Henson', 'Guy Boertje']
   s.email       = ['lee.m.henson@gmail.com', 'guyboertje@gmail.com']
@@ -15,59 +15,33 @@ Gem::Specification.new do |s|
     Rakefile
     euston-daemons.gemspec
     lib/euston-daemons.rb
-    lib/euston-daemons/command_processor_daemon/config/environment.rb
-    lib/euston-daemons/command_processor_daemon/lib/clients/command_handler.rb
-    lib/euston-daemons/command_processor_daemon/lib/command_handlers/retry_failed_message.rb
-    lib/euston-daemons/command_processor_daemon/lib/daemon.rb
-    lib/euston-daemons/command_processor_daemon/lib/mongo_models/failed_message.rb
-    lib/euston-daemons/command_processor_daemon/rake_task.rb
     lib/euston-daemons/euston/daemon.rb
-    lib/euston-daemons/euston/daemon_client.rb
     lib/euston-daemons/euston/daemon_component.rb
+    lib/euston-daemons/euston/daemon_component_host.rb
     lib/euston-daemons/euston/daemon_environment.rb
-    lib/euston-daemons/event_processor_daemon/config/environment.rb
-    lib/euston-daemons/event_processor_daemon/lib/clients/event_handler.rb
-    lib/euston-daemons/event_processor_daemon/lib/daemon.rb
-    lib/euston-daemons/event_processor_daemon/lib/event_handlers/message_failure.rb
-    lib/euston-daemons/event_processor_daemon/rake_task.rb
-    lib/euston-daemons/message_buffer_daemon/config/environment.rb
-    lib/euston-daemons/message_buffer_daemon/lib/clients/command_buffer_cleanup.rb
-    lib/euston-daemons/message_buffer_daemon/lib/clients/command_buffer_publisher.rb
-    lib/euston-daemons/message_buffer_daemon/lib/clients/command_logger.rb
-    lib/euston-daemons/message_buffer_daemon/lib/clients/euston_exchange_accessors.rb
-    lib/euston-daemons/message_buffer_daemon/lib/clients/event_buffer_cleanup.rb
-    lib/euston-daemons/message_buffer_daemon/lib/clients/event_buffer_publisher.rb
-    lib/euston-daemons/message_buffer_daemon/lib/clients/event_logger.rb
-    lib/euston-daemons/message_buffer_daemon/lib/clients/event_store_dispatcher.rb
-    lib/euston-daemons/message_buffer_daemon/lib/clients/message_buffer_cleanup.rb
-    lib/euston-daemons/message_buffer_daemon/lib/clients/message_buffer_publisher.rb
-    lib/euston-daemons/message_buffer_daemon/lib/clients/message_logger.rb
-    lib/euston-daemons/message_buffer_daemon/lib/clients/mongo_model_accessors.rb
-    lib/euston-daemons/message_buffer_daemon/lib/daemon.rb
-    lib/euston-daemons/message_buffer_daemon/lib/mongo_models/command_buffer.rb
-    lib/euston-daemons/message_buffer_daemon/lib/mongo_models/command_log.rb
-    lib/euston-daemons/message_buffer_daemon/lib/mongo_models/event_buffer.rb
-    lib/euston-daemons/message_buffer_daemon/lib/mongo_models/event_log.rb
-    lib/euston-daemons/message_buffer_daemon/lib/mongo_models/message_buffer.rb
-    lib/euston-daemons/message_buffer_daemon/lib/mongo_models/message_log.rb
-    lib/euston-daemons/message_buffer_daemon/rake_task.rb
+    lib/euston-daemons/euston/exceptions.rb
+    lib/euston-daemons/euston/stopwatch.rb
+    lib/euston-daemons/pipeline/config/environment.rb
+    lib/euston-daemons/pipeline/lib/command_logger/component.rb
+    lib/euston-daemons/pipeline/lib/command_logger/log.rb
+    lib/euston-daemons/pipeline/lib/command_processor/component.rb
+    lib/euston-daemons/pipeline/lib/command_processor/default_handlers/retry_failed_message.rb
+    lib/euston-daemons/pipeline/lib/command_processor/failed_message.rb
+    lib/euston-daemons/pipeline/lib/daemon.rb
+    lib/euston-daemons/pipeline/lib/event_processor/component.rb
+    lib/euston-daemons/pipeline/lib/event_processor/default_handlers/message_failure.rb
+    lib/euston-daemons/pipeline/lib/event_store_dispatcher/component.rb
+    lib/euston-daemons/pipeline/lib/message_buffer/buffer.rb
+    lib/euston-daemons/pipeline/lib/message_buffer/component.rb
+    lib/euston-daemons/pipeline/lib/snapshotter/component.rb
+    lib/euston-daemons/pipeline/rake_task.rb
     lib/euston-daemons/rake_task.rb
     lib/euston-daemons/rake_tasks.rb
-    lib/euston-daemons/snapshot_daemon/lib/clients/snapshotter.rb
     lib/euston-daemons/version.rb
-    sample/Rakefile
-    sample/amqp_config.yml
-    sample/command_handlers.rb
-    sample/command_processor_daemon_config.yml
-    sample/event_handlers.rb
-    sample/event_processor_daemon_config.yml
-    sample/message_buffer_daemon_config.yml
-    sample/mongoid_config.yml
-    sample/pids/.placeholder
-    spec/daemons/command_buffer_publisher_spec.rb
-    spec/daemons/command_handler_spec.rb
-    spec/daemons/event_handler_spec.rb
-    spec/daemons/snapshot_client_spec.rb
+    spec/daemons/command_processor_spec.rb
+    spec/daemons/event_processor_spec.rb
+    spec/daemons/message_buffer_spec.rb
+    spec/daemons/snapshotter_spec.rb
     spec/spec_helper.rb
     spec/support/factories/commands.rb
     spec/support/factories/commit.rb
@@ -87,15 +61,16 @@ Gem::Specification.new do |s|
   s.add_dependency              'activemodel',        '~> 3.0.0'
   s.add_dependency              'activesupport',      '~> 3.0.0'
   s.add_dependency              'erb-yaml',           '~> 1.0.0'
-  s.add_dependency              'euston',             '~> 1.1.0'
-  s.add_dependency              'euston-eventstore',  '~> 1.1.0'
-  s.add_dependency              'euston-rabbitmq',    '~> 1.1.0'
+  s.add_dependency              'euston',             '~> 1.2.0'
+  s.add_dependency              'euston-eventstore',  '~> 1.2.0'
+  s.add_dependency              'euston-rabbitmq',    '~> 1.2.0'
   s.add_dependency              'hollywood',          '~> 1.0.0'
   s.add_dependency              'i18n',               '~> 0.5.0'
+  s.add_dependency              'macaddr',            '~> 1.4.0'
   s.add_dependency              'safely',             '~> 0.3.0'
 
   if RUBY_PLATFORM.to_s == 'java'
-    s.add_dependency            'jessica',            '~> 1.0.0'
+    s.add_dependency            'hot_bunnies',        '~> 1.3.0.pre1'
     s.add_dependency            'jmongo',             '~> 1.1.0'
   end
 

data/lib/euston-daemons/euston/daemon.rb

@@ -16,6 +16,7 @@ module Euston
       start_components
       wait_for_shutdown_event
       stop_components
+      post_shutdown_cleanup
       report_shutdown_reasons
     end
 
@@ -26,10 +27,15 @@ module Euston
     end
 
     def register_component name, klass
-      component = Euston::DaemonComponent.new @log, name, klass, self
+      component = Euston::DaemonComponentHost.new @log, name, klass, self
       component.when(:fatal_error_in_component => method(:shutdown))
 
       @components << component
+      component
+    end
+
+    def post_shutdown_cleanup
+      # virtual
     end
 
     def register_components

data/lib/euston-daemons/euston/daemon_component.rb

@@ -1,65 +1,25 @@
 module Euston
   class DaemonComponent
     include Hollywood
+    include Daemons::Exceptions
 
-    def initialize logger, name, client, owner
-      @log, @name, @client, @owner = logger, name, client, owner
-    end
-
-    attr_reader :name
-
-    def start
-      @thread = Thread.new do
-        while_thread_is_running do
-          begin
-            @client.run
-          rescue => e
-            Thread.current[:exception] = e
-          end
-
-          report_exceptions
-        end
-      end
+    def run
+      next_iteration unless stopped
     end
 
     def stop
-      @client.stop
-      @thread[:stop] = true
-    end
-
-    def thread_state
-      @thread.status
+      @stopped = true
+      @channel.close unless @channel.nil? || !@channel.respond_to?(:close)
     end
 
     private
 
-    def report_exceptions
-      exception = Thread.current[:exception]
-
-      unless exception.nil? || exception.to_s =~ /clean channel shutdown/
-        if exception.respond_to? :cause           # its a serious (Java?) exception
-          Thread.current[:stop] = true
-          callback :fatal_error_in_component, exception
-        else
-          Safely.report! exception
-        end
-      end
-    end
-
-    def thread_is_stopping?
-      Thread.current[:stop]
+    def next_iteration
+      # abstract
     end
 
-    def wait_time
-      @wait_time ||= 0.2
-    end
-
-    def while_thread_is_running
-      until thread_is_stopping? do
-        yield
-
-        sleep wait_time
-      end
+    def stopped
+      @stopped ||= false
     end
   end
 end

data/lib/euston-daemons/euston/daemon_component_host.rb

@@ -0,0 +1,66 @@
+module Euston
+  class DaemonComponentHost
+    include Hollywood
+    include Daemons::Exceptions
+
+    def initialize logger, name, client, owner
+      @log, @name, @client, @owner = logger, name, client, owner
+    end
+
+    attr_reader :name
+    attr_writer :wait_time
+
+    def start
+      @thread = Thread.new do
+        while_thread_is_running do
+          begin
+            @client.run
+          rescue => e
+            Thread.current[:exception] = e
+          end
+
+          report_exceptions
+        end
+      end
+    end
+
+    def stop
+      @client.stop
+      @thread[:stop] = true
+    end
+
+    def thread_state
+      @thread.status
+    end
+
+    private
+
+    def report_exceptions
+      exception = Thread.current[:exception]
+      return if exception.nil? || ignorable_exception?(exception)
+
+      if exception.respond_to? :cause           # its a serious (Java?) exception
+        Thread.current[:stop] = true
+        callback :fatal_error_in_component, exception
+      else
+        Safely.report! exception
+      end
+    end
+
+    def thread_is_stopping?
+      Thread.current[:stop]
+    end
+
+    def wait_time
+      @wait_time ||= 0.2
+    end
+
+    def while_thread_is_running
+      until thread_is_stopping? do
+        yield
+
+        sleep wait_time
+      end
+    end
+  end
+end

data/lib/euston-daemons/euston/daemon_environment.rb

@@ -11,7 +11,11 @@ module Euston
     private
 
     def setup_amqp config
-      AMQP.settings.merge! config
+      class << self.class
+        attr_accessor :amqp_connection
+      end
+
+      self.class.amqp_connection = HotBunnies.connect config
     end
 
     def setup_euston event_store_database_name
@@ -29,14 +33,15 @@ module Euston
     end
 
     def setup_mongo config
-      connection = Mongo::Connection.new config[:host], config[:port], :safe => config[:safe], :logger => @logger
+      # logger arg will not work until jmongo forwards it correctly
+      connection = Mongo::Connection.from_uri config[:uri], :logger => @logger
 
       class << self.class
-        attr_accessor :event_store_mongodb, :read_model_mongodb
+        attr_accessor :event_store_mongodb, :projections_mongodb
       end
 
-      self.class.event_store_mongodb  = Mongo::DB.new config[:event_store_database], connection
-      self.class.read_model_mongodb   = Mongo::DB.new config[:read_model_database],  connection
+      self.class.event_store_mongodb = Mongo::DB.new config[:event_store_database], connection
+      self.class.projections_mongodb = Mongo::DB.new config[:projections_database], connection
     end
 
     def setup_safely hoptoad_key

data/lib/euston-daemons/euston/exceptions.rb

@@ -0,0 +1,9 @@
+module Euston
+  module Daemons
+    module Exceptions
+      def ignorable_exception? exception
+        exception.to_s =~ /clean channel shutdown|clean connection shutdown/
+      end
+    end
+  end
+end

data/lib/euston-daemons/euston/stopwatch.rb

@@ -0,0 +1,15 @@
+module Euston
+  class Stopwatch
+    include Hollywood
+
+    def time decimal_places = 3, &block
+      start = Time.now.to_f
+      result = yield block
+      finish = Time.now.to_f
+      elapsed = finish - start
+      elapsed = elapsed.round(decimal_places) unless decimal_places.nil?
+      callback :finished, elapsed
+      result
+    end
+  end
+end

data/lib/euston-daemons/pipeline/config/environment.rb

@@ -0,0 +1,74 @@
+require 'i18n'
+require 'hot_bunnies'
+require 'macaddr'
+require 'euston-rabbitmq'
+
+require 'euston-daemons/pipeline/lib/command_processor/failed_message'
+require 'euston-daemons/pipeline/lib/command_processor/default_handlers/retry_failed_message'
+require 'euston-daemons/pipeline/lib/command_processor/component'
+require 'euston-daemons/pipeline/lib/command_logger/log'
+require 'euston-daemons/pipeline/lib/command_logger/component'
+require 'euston-daemons/pipeline/lib/event_processor/default_handlers/message_failure'
+require 'euston-daemons/pipeline/lib/event_processor/component'
+require 'euston-daemons/pipeline/lib/event_store_dispatcher/component'
+require 'euston-daemons/pipeline/lib/message_buffer/buffer'
+require 'euston-daemons/pipeline/lib/message_buffer/component'
+require 'euston-daemons/pipeline/lib/snapshotter/component'
+require 'euston-daemons/pipeline/lib/daemon'
+
+module Euston
+  module Daemons
+    module Pipeline
+      class DaemonEnvironment < Euston::DaemonEnvironment
+        class << self
+          attr_accessor :command_processors,
+                        :command_handler_namespaces,
+                        :command_loggers,
+                        :event_handler_namespaces,
+                        :event_processors,
+                        :event_store_dispatchers,
+                        :event_store_dispatcher_wait_time,
+                        :message_buffers,
+                        :message_buffer_wait_time,
+                        :snapshotters,
+                        :snapshotter_wait_time,
+                        :snapshot_threshold
+        end
+
+        def initialize data
+          @logger             = data[:logger]
+          @daemon_config      = data[:daemon_config]
+          @i18n_locales_path  = @daemon_config[:i18n_locales_path]
+
+          @amqp_config  = ErbYaml.read data[:amqp_config_path],   data[:environment]
+          @mongo_config = ErbYaml.read data[:mongo_config_path],  data[:environment]
+
+          self.class.command_handler_namespaces       = data[:command_handler_namespaces]
+          self.class.event_handler_namespaces         = data[:event_handler_namespaces]
+
+          self.class.command_processors               = @daemon_config[:command_processors].to_i
+          self.class.command_loggers                  = @daemon_config[:command_loggers].to_i
+          self.class.event_processors                 = @daemon_config[:event_processors].to_i
+          self.class.event_store_dispatchers          = @daemon_config[:event_store_dispatchers].to_i
+          self.class.event_store_dispatcher_wait_time = @daemon_config[:event_store_dispatcher_wait_time].to_f
+          self.class.message_buffers                  = @daemon_config[:message_buffers].to_i
+          self.class.message_buffer_wait_time         = @daemon_config[:message_buffer_wait_time].to_f
+          self.class.snapshotters                     = @daemon_config[:snapshotters].to_i
+          self.class.snapshotter_wait_time            = @daemon_config[:snapshotter_wait_time].to_f
+          self.class.snapshot_threshold               = @daemon_config[:snapshot_threshold].to_i
+        end
+
+        def setup
+          setup_safely @daemon_config[:hoptoad_key]
+          setup_amqp @amqp_config
+          setup_mongo @mongo_config
+          setup_euston @mongo_config[:event_store_database]
+
+          I18n.load_path += Dir[@i18n_locales_path] unless @i18n_locales_path.nil?
+
+          self.class
+        end
+      end
+    end
+  end
+end

data/lib/euston-daemons/pipeline/lib/command_logger/component.rb

@@ -0,0 +1,54 @@
+module Euston
+  module Daemons
+    module Pipeline
+      module CommandLogger
+        class Component < Euston::DaemonComponent
+          extend RabbitMq::Exchanges
+          extend RabbitMq::Queues
+
+          def initialize channel, id = 1, logger = Euston::NullLogger.instance
+            @id = id
+            @log = logger
+            @channel = channel
+            @channel.prefetch = 1
+            @command_log = Log.new DaemonEnvironment.event_store_mongodb
+            @exchange = self.class.get_exchange channel, :commands
+            @queue = self.class.get_queue channel, @command_log.name
+            @queue.bind @exchange, :routing_key => "#{@exchange.name}.#"
+
+            @queue.when(:message_decode_failed => method(:log_failure),
+                        :message_failed        => method(:message_failed),
+                        :message_received      => method(:write_message_to_log))
+
+            @consumer = @queue.consumer
+          end
+
+          private
+
+          def log_failure message, error
+            return if ignorable_exception? error
+
+            text = "Command logger #{@id} failed.\n    [Error] #{error.message}\n    [Payload] #{message}"
+            err = Euston::RabbitMq::MessageDecodeFailedError.new text
+            err.set_backtrace error.backtrace
+
+            Safely.report! err
+          end
+
+          def message_failed message, error, reactive_message
+            log_failure message, error
+            reactive_message.ack
+          end
+
+          def next_iteration
+            @queue.safe_subscribe_with_timeout @consumer
+          end
+
+          def write_message_to_log message
+            @command_log.write_command message
+          end
+        end
+      end
+    end
+  end
+end

data/lib/euston-daemons/pipeline/lib/command_logger/log.rb

@@ -0,0 +1,31 @@
+module Euston
+  module Daemons
+    module Pipeline
+      module CommandLogger
+        class Log
+          def initialize mongodb
+            @name = 'command_log'
+            mongodb.create_collection @name unless mongodb.collection_names.include? @name
+
+            @collection = mongodb.collection @name
+          end
+
+          attr_reader :name
+
+          def write_command command
+            timestamp = Time.now.to_f
+
+            doc = { '_id'                   => command[:headers][:id],
+                    'type'                  => command[:headers][:type],
+                    'version'               => command[:headers][:version],
+                    'timestamp'             => timestamp,
+                    'timestamp_for_humans'  => Time.at(timestamp),
+                    'json'                  => ActiveSupport::JSON.encode(command) }
+
+            @collection.insert doc
+          end
+        end
+      end
+    end
+  end
+end

data/lib/euston-daemons/pipeline/lib/command_processor/component.rb

@@ -0,0 +1,50 @@
+module Euston
+  module Daemons
+    module Pipeline
+      module CommandProcessor
+        class Component < Euston::DaemonComponent
+          include Euston::CommandHandlerPrivateMethodNames
+
+          def initialize channel, handlers, id = 1, logger = Euston::NullLogger.instance
+            @channel = channel
+            @channel.prefetch = 1
+            @handlers = handlers
+            @id = id
+            @log = logger
+            @process_method = method(:process_message)
+            @stopwatch = Stopwatch.new.when(:finished => method(:log_elapsed_time))
+          end
+
+          private
+
+          def next_iteration
+            Euston::RabbitMq::RetryingSubscription.new(@channel, :command_handlers, @log)
+              .when(:message_received => @process_method)
+              .subscribe
+          end
+
+          def process_message message
+            @stopwatch.time do
+              command_headers = CommandHeaders.from_hash(message[:headers]).freeze
+              command_body = message[:body].freeze
+
+              handler_type = command_headers.type.to_s.camelize.to_sym
+              reference = @handlers.find { |reference| reference.name == handler_type }
+
+              if reference.nil?
+                Euston::CommandBus.publish command_headers, command_body, @log
+              else
+                handler_method_name = self.class.command_handler_method_name command_headers.version
+                reference.handler.new.send handler_method_name, command_headers, command_body
+              end
+            end
+          end
+
+          def log_elapsed_time elapsed_time
+            @log.debug "Command handler client #{@id} processed a message in #{elapsed_time} sec(s)"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/euston-daemons/pipeline/lib/command_processor/default_handlers/retry_failed_message.rb

@@ -0,0 +1,34 @@
+module Euston
+  module Daemons
+    module Pipeline
+      module CommandProcessor
+        module DefaultHandlers
+          class RetryFailedMessage
+            include Euston::CommandHandler
+
+            version 1 do |headers, command|
+              failed_messages = FailedMessage.new DaemonEnvironment.event_store_mongodb
+              message = failed_messages.get_by_id command[:id]
+
+              unless message.nil?
+                routing_key = message['routing_key']
+                exchange = routing_key.split('.').first
+
+                headers = message['headers'].merge(:id => command[:id],
+                                                   :type => message['type'],
+                                                   :version => message['version'] )
+
+                retry_message = { :headers => headers, :body => message['body'] }
+
+                buffer = MessageBuffer::Buffer.new DaemonEnvironment.event_store_mongodb
+                buffer.enqueue exchange, retry_message
+
+                failed_messages.remove_by_id command[:id]
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/euston-daemons/pipeline/lib/command_processor/failed_message.rb

@@ -0,0 +1,36 @@
+module Euston
+  module Daemons
+    module Pipeline
+      module CommandProcessor
+        class FailedMessage
+          def initialize mongodb
+            name = 'failed_messages'
+            mongodb.create_collection name unless mongodb.collection_names.include? name
+
+            @collection = mongodb.collection name
+            @collection.ensure_index [ ['failure_timestamp', Mongo::ASCENDING] ], :unique => false, :name => 'failed_messages_failure_timestamp_index'
+          end
+
+          def get_by_id id
+            @collection.find_one({ '_id' => id })
+          end
+
+          def find_all
+            @collection.find({}, { :sort => [ 'failure_timestamp', Mongo::DESCENDING ] })
+          end
+
+          def log_failure failure
+            failure.recursive_stringify_keys!
+            failure['_id'] = failure.delete 'message_id'
+
+            @collection.save(failure, :safe => { :fsync => true })
+          end
+
+          def remove_by_id id
+            @collection.remove({ '_id' => id }, :safe => { :fsync => true })
+          end
+        end
+      end
+    end
+  end
+end