euston-daemons 1.0.5 → 1.2.1

data/lib/euston-daemons/pipeline/lib/event_processor/default_handlers/message_failure.rb

@@ -0,0 +1,30 @@
+module Euston
+  module Daemons
+    module Pipeline
+      module EventProcessor
+        module DefaultHandlers
+          class MessageFailure
+            include Euston::EventHandler
+
+            subscribes :message_failed, 1 do |headers, event|
+              headers = event[:message][:headers].dup
+              failure = { :message_id         => headers.delete(:id),
+                          :type               => headers.delete(:type),
+                          :version            => headers.delete(:version),
+                          :message_timestamp  => headers.delete(:timestamp),
+                          :routing_key        => event[:routing_key],
+                          :body               => event[:message][:body],
+                          :headers            => headers,
+                          :error              => event[:error],
+                          :backtrace          => event[:backtrace],
+                          :failure_timestamp  => Time.now.to_f }
+
+              failed_messages = CommandProcessor::FailedMessage.new DaemonEnvironment.event_store_mongodb
+              failed_messages.log_failure failure
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/euston-daemons/pipeline/lib/event_store_dispatcher/component.rb

@@ -0,0 +1,68 @@
+module Euston
+  module Daemons
+    module Pipeline
+      module EventStoreDispatcher
+        class Component < DaemonComponent
+          extend RabbitMq::Exchanges
+
+          def initialize channel, id = 1, log = Euston::NullLogger.instance
+            @channel = channel
+            @channel.tx_select
+            @id = id
+            @log = log
+            @event_store = DaemonEnvironment.event_store
+            @stopwatch = Stopwatch.new.when(:finished => method(:log_elapsed_time))
+            @commands_exchange = self.class.get_exchange channel, :commands
+            @events_exchange = self.class.get_exchange channel, :events
+            @buffer = MessageBuffer::Buffer.new DaemonEnvironment.event_store_mongodb
+          end
+
+          private
+
+          def log_elapsed_time elapsed_time
+            @log.debug "Event store dispatcher #{@id} dispatched #{@commits_dispatched} commit(s) in #{elapsed_time} sec(s)" if @commits_dispatched > 0
+          end
+
+          def next_iteration
+            @commits_dispatched = 0
+
+            @stopwatch.time do
+              begin
+                @event_store.take_ownership_of_undispatched_commits @id
+                @commits = @event_store.get_undispatched_commits @id
+
+                begin
+                  @commits.each do |commit|
+                    commit.events.each do |event|
+                      hash = event.to_hash
+                      @events_exchange.publish ActiveSupport::JSON.encode(hash), self.class.default_publish_options.merge(:routing_key => "events.#{hash[:headers][:type]}")
+                    end
+
+                    commit.commands.each do |command|
+                      hash = command.to_hash
+
+                      if hash[:headers][:dispatch_at].nil?
+                        @commands_exchange.publish ActiveSupport::JSON.encode(hash), self.class.default_publish_options.merge(:routing_key => "commands.#{hash[:headers][:type]}")
+                      else
+                        @buffer.enqueue :commands, hash, hash[:headers][:dispatch_at]
+                      end
+                    end
+                  end
+
+                  @channel.tx_commit
+                rescue StandardError => e
+                  @channel.tx_rollback
+                  raise e
+                end
+
+                @event_store.mark_commits_as_dispatched @commits
+
+                @commits_dispatched += @commits.size
+              end until stopped || @commits.empty?
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/euston-daemons/pipeline/lib/message_buffer/buffer.rb

@@ -0,0 +1,85 @@
+module Euston
+  module Daemons
+    module Pipeline
+      module MessageBuffer
+        class Buffer
+          def initialize mongodb
+            name = 'message_buffer'
+            mongodb.create_collection name unless mongodb.collection_names.include? name
+
+            @name = name
+            @collection = mongodb.collection name
+
+            @collection.ensure_index [ ['message_id', Mongo::ASCENDING] ],
+                                       :unique  => false,
+                                       :name    => "#{name}_message_id_index"
+
+            @collection.ensure_index [ ['component_id', Mongo::ASCENDING],
+                                       ['dispatch_at',  Mongo::ASCENDING] ],
+                                       :unique  => false,
+                                       :name    => "#{name}_component_id_dispatch_at_index"
+          end
+
+          attr_reader :name
+
+          def delete_dispatched_messages component_id
+            @collection.remove({ 'component_id' => component_id }, :multi => true)
+          end
+
+          def enqueue exchange, message, dispatch_at = nil
+            messages = message
+            messages = [{ :hash => message, :dispatch_at => dispatch_at }] unless messages.is_a? Array
+
+            messages = messages.map do |m|
+              message_is_well_formed = m.is_a?(Hash) && m.has_key?(:hash) && m.has_key?(:dispatch_at)
+              m = { :hash => m, :dispatch_at => dispatch_at } unless message_is_well_formed
+              map_to_document exchange, m
+            end
+
+            @collection.insert(messages) unless messages.empty?
+          end
+
+          def find_dispatchable_messages component_id
+            query   = { 'component_id' => component_id }
+            fields  = ['exchange', 'type', 'json']
+            sort    = [ 'dispatch_at', Mongo::ASCENDING ]
+
+            @collection.find query, :fields => fields, :sort => sort
+          end
+
+          def get_by_id id
+            @collection.find_one 'message_id' => id
+          end
+
+          def take_ownership_of_dispatchable_messages component_id
+            new_messages_eligible_for_dispatch  = { 'component_id'  => '',
+                                                    'dispatch_at'   => { '$lte' => Time.now.to_f } }
+
+            messages_stuck_in_other_components  = { 'component_id'  => { '$ne'  => '' },
+                                                    'dispatch_at'   => { '$lte' => Time.now.to_f - 60 } }
+
+            query = { '$or'   => [ new_messages_eligible_for_dispatch, messages_stuck_in_other_components ] }
+            doc   = { '$set'  => { 'component_id' => component_id } }
+
+            @collection.update query, doc, :multi => true
+          end
+
+          private
+
+          def map_to_document exchange, message
+            hash        = message[:hash]
+            dispatch_at = (message[:dispatch_at] || Time.now.to_f).to_f
+
+            { 'message_id'              => hash[:headers][:id],
+              'exchange'                => exchange,
+              'type'                    => hash[:headers][:type],
+              'component_id'            => '',
+              'dispatch_at'             => dispatch_at,
+              'dispatch_at_for_humans'  => Time.at(dispatch_at),
+              'json'                    => ActiveSupport::JSON.encode(hash) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/euston-daemons/pipeline/lib/message_buffer/component.rb

@@ -0,0 +1,59 @@
+module Euston
+  module Daemons
+    module Pipeline
+      module MessageBuffer
+        class Component < Euston::DaemonComponent
+          extend RabbitMq::Exchanges
+
+          def initialize channel, id = 1, logger = Euston::NullLogger.instance
+            @channel = channel
+            @channel.tx_select
+            @id = "message_buffer #{id}"
+            @log = logger
+            @buffer = Buffer.new DaemonEnvironment.event_store_mongodb
+            @stopwatch = Stopwatch.new.when(:finished => method(:log_elapsed_time))
+          end
+
+          private
+
+          def dispatch_due_messages
+            @dispatched_count = 0
+
+            @stopwatch.time do
+              @buffer.take_ownership_of_dispatchable_messages @id
+
+              begin
+                @buffer.find_dispatchable_messages(@id).each do |message|
+                  exchange = self.class.get_exchange @channel, message['exchange']
+                  exchange.publish message['json'], self.class.default_publish_options.merge(:routing_key => "#{exchange.name}.#{message['type']}")
+                  @dispatched_count += 1
+                end
+
+                @channel.tx_commit
+              rescue StandardError => e
+                @channel.tx_rollback
+                raise e
+              end
+
+              @buffer.delete_dispatched_messages @id
+            end
+
+            @dispatched_count
+          end
+
+          def log_elapsed_time elapsed_time
+            @log.debug "Message buffer #{@id} dispatched #{@dispatched_count} message(s) in #{elapsed_time} sec(s)" if @dispatched_count > 0
+          end
+
+          def next_iteration
+            @messages_dispatched = 0
+
+            begin
+              @messages_dispatched = dispatch_due_messages
+            end until stopped || @messages_dispatched.zero?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/euston-daemons/pipeline/lib/snapshotter/component.rb

@@ -0,0 +1,48 @@
+module Euston
+  module Daemons
+    module Pipeline
+      module Snapshotter
+        class Component < Euston::DaemonComponent
+          def initialize event_store, threshold, id = 1, logger = Euston::NullLogger.instance
+            @event_store = event_store
+            @threshold = threshold
+            @id = id
+            @log = logger
+          end
+
+          private
+
+          def next_iteration
+            begin
+              stream_heads = @event_store.get_streams_to_snapshot @threshold
+              @log.debug "Found #{stream_heads.length} stream(s) eligible for snapshotting (threshold is #{@threshold})" if stream_heads.any?
+
+              stream_heads.each do |stream_head|
+                pair = @event_store.get_snapshot_stream_pair stream_head.stream_id
+
+                loader = RabbitMq::ConstantLoader.new
+                loader.when(:hit => ->(klass) { take_snapshot klass, pair },
+                            :miss => ->(type) { Safely.report! "Snapshotter was unable to find a class: #{type}" })
+
+                loader.load pair.stream.committed_headers[:aggregate_type]
+              end
+            end until stopped || stream_heads.empty?
+          end
+
+          def take_snapshot klass, pair
+            instance = klass.hydrate pair.stream, pair.snapshot
+            snapshot = instance.take_snapshot
+            snapshot = EventStore::Snapshot.new pair.stream.stream_id,
+                                                pair.stream.stream_revision,
+                                                snapshot[:payload],
+                                                :version => snapshot[:version]
+
+            @log.debug "Writing snapshot: #{snapshot.inspect}"
+
+            @event_store.add_snapshot snapshot
+          end
+        end
+      end
+    end
+  end
+end

data/lib/euston-daemons/pipeline/rake_task.rb

@@ -0,0 +1,49 @@
+module Euston
+  class PipelineRakeTask < Euston::Daemons::RakeTask
+    attr_accessor :amqp_config_path,
+                  :command_handler_namespaces,
+                  :daemon_config_path,
+                  :event_handler_namespaces,
+                  :mongo_config_path,
+                  :user_defined_components
+
+    def initialize environment
+      super(environment, :pipeline_daemon)
+    end
+
+    def before_creating_task
+      @daemon_path = File.expand_path(File.dirname __FILE__) + File::SEPARATOR
+      @daemon_class = 'Euston::Daemons::Pipeline::Daemon'
+    end
+
+    def load_daemon_config_file
+      @data[:daemon_config] = ErbYaml.read @daemon_config_path, @data[:environment]
+    end
+
+    def initialize_settings
+      @logger.debug "AMQP config path: #{@amqp_config_path}"
+      @data[:amqp_config_path] = @amqp_config_path
+
+      @logger.debug "Command handler namespaces: #{@command_handler_namespaces}"
+      @data[:command_handler_namespaces] = @command_handler_namespaces
+
+      @logger.debug "Daemon config path: #{@daemon_config_path}"
+
+      @logger.debug "Event handler namespaces: #{@event_handler_namespaces}"
+      @data[:event_handler_namespaces] = @event_handler_namespaces
+
+      @logger.debug "Mongo config path: #{@mongo_config_path}"
+      @data[:mongo_config_path] = @mongo_config_path
+
+      @logger.debug "User-defined components: #{@user_defined_components.count}"
+      @data[:user_defined_components] = @user_defined_components.to_a.flatten
+    end
+
+    def load_environment
+      @logger.debug "Loading environment"
+      require 'euston-daemons/pipeline/config/environment'
+
+      Euston::Daemons::Pipeline::DaemonEnvironment.new(@data).setup
+    end
+  end
+end

data/lib/euston-daemons/rake_task.rb

@@ -15,11 +15,8 @@ module Euston
       # Daemon class. Must be supplied as a string.
       attr_accessor :daemon_class
 
-      # Log file path
-      #
-      # default:
-      #   /var/log/euston.log
-      attr_accessor :log_path
+      # Callable. Receives the daemon environment object to allow the user to perform other related config operations.
+      attr_accessor :post_setup_callback
 
       # Use verbose output. If this is set to true, the task will print the
       # executed command to stdout.
@@ -28,65 +25,51 @@ module Euston
       #   true
       attr_accessor :verbose
 
-      def initialize(*args)
-        @name = args.shift || :eustondaemon
-        @log_path = '/var/log/euston.log'
+      def initialize environment, name = :euston_daemon
+        @name = name
         @verbose = true
-
-        Object.const_set :EUSTON_ENV, get_environment unless Object.const_defined? :EUSTON_ENV
-
-        @pid_path = pid_path_for_env + "#{@name}.pid"
+        @environment = validate_environment environment
+        @data = { :environment => @environment }
 
         yield self if block_given?
         send :before_creating_task if respond_to? :before_creating_task
 
         desc("Run a Euston daemon") unless ::Rake.application.last_comment
-
         task name do
           RakeFileUtils.send(:verbose, verbose) do
+            load_daemon_config_file
             initialize_logger
+            initialize_settings
+            write_pid_file
             log_startup
-            initialize_paths
-            load_framework
-            load_environment
+            env = load_environment
+            post_setup_callback.call env unless post_setup_callback.nil?
             launch_and_wait_for_exit
             log_shutdown
+            remove_pid_file
           end
         end
       end
 
       private
 
-      def get_environment
-        environment = ENV['EUSTON_ENV'].to_s.downcase.to_sym
-        environments = [:development, :test, :staging, :production]
-        environment = :development unless environments.include? environment
-        environment
-      end
+      def initialize_logger
+        config = @data[:daemon_config]
+        log_path = config[:log_path]
 
-      def pid_path_for_env
-        case get_environment
-        when :development, :test
-          '~/var/run/euston/'
-        else
-          '/var/run/euston/'
-        end
-      end
+        raise "Required log path does not exist: #{log_path}" unless Dir.exist? log_path
 
-      def initialize_logger
-        Object.const_set :EUSTON_LOG, Logger.new(@log_path.gsub(/\.log$/, ".#{EUSTON_ENV}.log"))
-      end
+        @data[:logger] = @logger = Logger.new(File.join log_path, "#{@name}.#{@environment}.log")
 
-      def initialize_paths
-        # virtual
+        begin
+          @logger.level = Logger.const_get config[:log_level].upcase.to_sym
+        rescue
+          @logger.level = Logger::DEBUG
+        end
       end
 
-      def rake_pid
-        if defined? Java
-          Java::JavaIo::File.new("/proc/self").canonical_file.name
-        else
-          File.readlink("/proc/self")
-        end
+      def initialize_settings
+        # virtual
       end
 
       def launch_and_wait_for_exit
@@ -97,26 +80,15 @@ module Euston
           ns = ns.const_get c.to_sym
         end
 
-        daemon = ns.new
-        create_pid_file
-        trap_exit_signals daemon
-        daemon.run
-        remove_pid_file
+        ns.new(@data).run
       end
 
-      def load_framework
-        EUSTON_LOG.debug "Loading framework"
-        require_rel 'framework'
-      end
-
-      def create_pid_file
-        File.open( @pid_path, 'w' ) { |f|
-          f.puts rake_pid
-        }
+      def load_daemon_config_file
+        # virtual
       end
 
-      def remove_pid_file
-        File.delete( @pid_path ) rescue Errno::ENOENT
+      def load_environment
+        # virtual
       end
 
       def log_shutdown
@@ -130,17 +102,42 @@ module Euston
       def print_log_banner banner
         border = '-' * banner.length
 
-        EUSTON_LOG.info ''
-        EUSTON_LOG.info border
-        EUSTON_LOG.info banner
-        EUSTON_LOG.info border
-        EUSTON_LOG.info ''
+        @logger.info ''
+        @logger.info border
+        @logger.info banner
+        @logger.info border
+        @logger.info ''
       end
 
-      def trap_exit_signals daemon
-        signals = %W(INT TERM) & Signal.list.keys
-        signals.each { |sig| sig.freeze }.freeze
-        signals.each { |sig| Signal.trap(sig) { daemon.queue.push(sig) } }
+      def remove_pid_file
+        @logger.debug "Deleting pid file at #{@pid_file}"
+
+        File.delete @pid_file rescue Errno::ENOENT
+      end
+
+      def validate_environment environment
+        environment = environment.to_s.downcase.to_sym
+        environments = [:development, :test, :staging, :production]
+        environment = :development unless environments.include? environment
+        environment
+      end
+
+      def write_pid_file
+        pid_path = @data[:daemon_config][:pid_path]
+
+        raise "Required pid path does not exist: #{pid_path}" unless Dir.exist? pid_path
+
+        @pid_file = File.join pid_path, "#{@name}.#{@environment}.pid"
+
+        if defined? Java
+          @pid = java.lang.management.ManagementFactory.getRuntimeMXBean().getName().split('@').first
+        else
+          @pid = File.readlink("/proc/self")
+        end
+
+        @logger.error "Writing pid #{@pid} to #{@pid_file}"
+
+        File.open(@pid_file, 'w') { |f| f.puts @pid }
       end
     end
   end