euston-daemons 1.0.4-java → 1.1.0-java

data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/command_buffer.rb

@@ -0,0 +1,11 @@
+module Euston
+  module MessageBufferDaemon
+    module MongoModel
+      class CommandBuffer < MessageBuffer
+        def initialize event_store_mongodb
+          super 'command_buffer', event_store_mongodb
+        end
+      end
+    end
+  end
+end

data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/command_log.rb

@@ -0,0 +1,11 @@
+module Euston
+  module MessageBufferDaemon
+    module MongoModel
+      class CommandLog < MessageLog
+        def initialize event_store_mongodb
+          super 'command_log', event_store_mongodb
+        end
+      end
+    end
+  end
+end

data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/event_buffer.rb

@@ -0,0 +1,11 @@
+module Euston
+  module MessageBufferDaemon
+    module MongoModel
+      class EventBuffer < MessageBuffer
+        def initialize event_store_mongodb
+          super 'event_buffer', event_store_mongodb
+        end
+      end
+    end
+  end
+end

data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/event_log.rb

@@ -0,0 +1,11 @@
+module Euston
+  module MessageBufferDaemon
+    module MongoModel
+      class EventLog < MessageLog
+        def initialize event_store_mongodb
+          super 'event_log', event_store_mongodb
+        end
+      end
+    end
+  end
+end

data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/message_buffer.rb

@@ -0,0 +1,57 @@
+module Euston
+  module MessageBufferDaemon
+    module MongoModel
+      class MessageBuffer
+        def initialize name, mongodb
+          mongodb.create_collection name unless mongodb.collection_names.include? name
+
+          @name = name
+          @collection = mongodb.collection name
+          @collection.ensure_index [ ['next_attempt', Mongo::ASCENDING] ], :unique => false, :name => "#{name}_next_attempt_index"
+        end
+
+        attr_reader :name
+
+        def buffer_new_message message, dispatched_at = nil
+          next_attempt = (dispatched_at || Time.now.to_f).to_f
+
+          @collection.save({ '_id' => message[:headers][:id],
+                             'type' => message[:headers][:type],
+                             'next_attempt' => next_attempt,
+                             'next_attempt_for_humans' => Time.at(next_attempt),
+                             'json' => ::ActiveSupport::JSON.encode(message) }, :safe => { :fsync => true })
+        end
+
+        def find_next_message
+          query = { 'next_attempt' => { '$lte' => Time.now.to_f } }
+          @collection.find_one(query)
+        end
+
+        def find_due_messages
+          query = { 'next_attempt' => { '$lte' => Time.now.to_f } }
+          order = [ 'next_attempt', Mongo::ASCENDING ]
+
+          @collection.find(query).sort(order)
+        end
+
+        def get_by_id id
+          @collection.find_one({ '_id' => id })
+        end
+
+        def set_next_attempt message
+          next_attempt = Time.now.to_f + 10
+
+          id  = { '_id' => message['_id'] }
+          doc = { '$set' => { 'next_attempt' => next_attempt,
+                              'next_attempt_for_humans' => Time.at(next_attempt) } }
+
+          @collection.update id, doc, :safe => { :fsync => true }
+        end
+
+        def remove_published_message id
+          @collection.remove({ '_id' => id }, :safe => { :fsync => true })
+        end
+      end
+    end
+  end
+end

data/lib/euston-daemons/message_buffer_daemon/lib/{read_model → mongo_models}/message_log.rb

@@ -1,24 +1,17 @@
 module Euston
   module MessageBufferDaemon
-    module ReadModel
+    module MongoModel
       class MessageLog
-        class << self
-          def commands
-            self.new "command_log", Euston::RabbitMq.event_store_mongodb
-          end
-
-          def events
-            self.new "event_log", Euston::RabbitMq.event_store_mongodb
-          end
-        end
-
         def initialize name, mongodb
           mongodb.create_collection name unless mongodb.collection_names.include? name
 
+          @name = name
           @collection = mongodb.collection name
           @collection.ensure_index [ ['timestamp', Mongo::ASCENDING] ], :unique => false, :name => "#{name}_timestamp_index"
         end
 
+        attr_reader :name
+
         def find_all
           @collection.find({}, { :sort => [ 'timestamp', Mongo::DESCENDING ] })
         end

data/lib/euston-daemons/message_buffer_daemon/rake_task.rb

@@ -2,8 +2,8 @@ module Euston
   class MessageBufferRakeTask < Euston::Daemons::RakeTask
     attr_accessor :amqp_config_path, :daemon_config_path, :mongoid_config_path
 
-    def initialize
-      super(:message_buffer_daemon)
+    def initialize environment
+      super(environment, :message_buffer_daemon)
     end
 
     def before_creating_task
@@ -11,20 +11,22 @@ module Euston
       @daemon_class = 'Euston::MessageBufferDaemon::Daemon'
     end
 
-    def initialize_paths
-      EUSTON_LOG.debug "AMQP config path: #{@amqp_config_path}"
-      Object.const_set :AMQP_CONFIG_PATH, @amqp_config_path
+    def initialize_settings
+      @logger.debug "AMQP config path: #{@amqp_config_path}"
+      @data[:amqp_config_path] = @amqp_config_path
 
-      EUSTON_LOG.debug "Daemon config path: #{@daemon_config_path}"
-      Object.const_set :DAEMON_CONFIG_PATH, @daemon_config_path
+      @logger.debug "Daemon config path: #{@daemon_config_path}"
+      @data[:daemon_config_path] = @daemon_config_path
 
-      EUSTON_LOG.debug "Mongoid config path: #{@mongoid_config_path}"
-      Object.const_set :MONGOID_CONFIG_PATH, @mongoid_config_path
+      @logger.debug "Mongoid config path: #{@mongoid_config_path}"
+      @data[:mongoid_config_path] = @mongoid_config_path
     end
 
     def load_environment
-      EUSTON_LOG.debug "Loading environment"
-      require_rel 'config/environment.rb'
+      @logger.debug "Loading environment"
+      require 'euston-daemons/message_buffer_daemon/config/environment'
+
+      Euston::MessageBufferDaemon::DaemonEnvironment.new(@data).setup
     end
   end
 end

data/lib/euston-daemons/rake_task.rb

@@ -15,11 +15,11 @@ module Euston
       # Daemon class. Must be supplied as a string.
       attr_accessor :daemon_class
 
-      # Log file path
-      #
-      # default:
-      #   /var/log/euston.log
-      attr_accessor :log_path
+      # Logger instance. Defaults to writing to /var/log/euston.log.
+      attr_accessor :logger
+
+      # # Path to write the daemon pid file to
+      attr_accessor :pid_path
 
       # Use verbose output. If this is set to true, the task will print the
       # executed command to stdout.
@@ -28,67 +28,48 @@ module Euston
       #   true
       attr_accessor :verbose
 
-      def initialize(*args)
-        @name = args.shift || :eustondaemon
-        @log_path = '/var/log/euston.log'
+      def initialize environment, name = :euston_daemon
+        @name = name
         @verbose = true
-
-        Object.const_set :EUSTON_ENV, get_environment unless Object.const_defined? :EUSTON_ENV
-
-        @pid_path = pid_path_for_env + "#{@name}.pid"
+        @environment = validate_environment environment
+        @data = { :environment => @environment }
 
         yield self if block_given?
         send :before_creating_task if respond_to? :before_creating_task
 
         desc("Run a Euston daemon") unless ::Rake.application.last_comment
-
         task name do
           RakeFileUtils.send(:verbose, verbose) do
             initialize_logger
+            write_pid_file
             log_startup
-            initialize_paths
-            load_framework
+            initialize_settings
             load_environment
             launch_and_wait_for_exit
             log_shutdown
+            remove_pid_file
           end
         end
       end
 
       private
 
-      def get_environment
-        environment = ENV['EUSTON_ENV'].to_s.downcase.to_sym
+      def validate_environment environment
+        environment = environment.to_s.downcase.to_sym
         environments = [:development, :test, :staging, :production]
         environment = :development unless environments.include? environment
         environment
       end
 
-      def pid_path_for_env
-        case get_environment
-        when :development, :test
-          '~/var/run/euston/'
-        else
-          '/var/run/euston/'
-        end
-      end
-
       def initialize_logger
-        Object.const_set :EUSTON_LOG, Logger.new(@log_path.gsub(/\.log$/, ".#{EUSTON_ENV}.log"))
+        @logger ||= Logger.new(STDOUT)
+        @data[:logger] = @logger
       end
 
-      def initialize_paths
+      def initialize_settings
         # virtual
       end
 
-      def rake_pid
-        if defined? Java
-          Java::JavaIo::File.new("/proc/self").canonical_file.name
-        else
-          File.readlink("/proc/self")
-        end
-      end
-
       def launch_and_wait_for_exit
         ns = Object
 
@@ -97,26 +78,7 @@ module Euston
           ns = ns.const_get c.to_sym
         end
 
-        daemon = ns.new
-        create_pid_file
-        trap_exit_signals daemon
-        daemon.run
-        remove_pid_file
-      end
-
-      def load_framework
-        EUSTON_LOG.debug "Loading framework"
-        require_rel 'framework'
-      end
-
-      def create_pid_file
-        File.open( @pid_path, 'w' ) { |f|
-          f.puts jvm_pid
-        }
-      end
-
-      def remove_pid_file
-        File.delete( @pid_path ) rescue Errno::ENOENT
+        ns.new(@data).run
       end
 
       def log_shutdown
@@ -130,17 +92,31 @@ module Euston
       def print_log_banner banner
         border = '-' * banner.length
 
-        EUSTON_LOG.info ''
-        EUSTON_LOG.info border
-        EUSTON_LOG.info banner
-        EUSTON_LOG.info border
-        EUSTON_LOG.info ''
+        @logger.info ''
+        @logger.info border
+        @logger.info banner
+        @logger.info border
+        @logger.info ''
+      end
+
+      def remove_pid_file
+        @logger.debug "Deleting pid file at #{@pid_file}"
+
+        File.delete @pid_file rescue Errno::ENOENT
       end
 
-      def trap_exit_signals daemon
-        signals = %W(INT TERM) & Signal.list.keys
-        signals.each { |sig| sig.freeze }.freeze
-        signals.each { |sig| Signal.trap(sig) { daemon.queue.push(sig) } }
+      def write_pid_file
+        @pid_file = File.join pid_path, "#{@name}.pid"
+
+        if defined? Java
+          @pid = Java::JavaIo::File.new("/proc/self").canonical_file.name
+        else
+          @pid = File.readlink("/proc/self")
+        end
+
+        @logger.debug "Writing pid #{@pid} to #{@pid_file}"
+
+        File.open(@pid_file, 'w') { |f| f.puts @pid }
       end
     end
   end

data/lib/euston-daemons/rake_tasks.rb

@@ -1,7 +1,7 @@
 require 'euston-daemons'
-require 'rake/tasklib'
 
-require_rel 'rake_task.rb'
-require_rel 'command_processor_daemon/rake_task.rb'
-require_rel 'event_processor_daemon/rake_task.rb'
-require_rel 'message_buffer_daemon/rake_task.rb'
+require 'rake/tasklib'
+require 'euston-daemons/rake_task'
+require 'euston-daemons/command_processor_daemon/rake_task'
+require 'euston-daemons/event_processor_daemon/rake_task'
+require 'euston-daemons/message_buffer_daemon/rake_task'

data/lib/euston-daemons/snapshot_daemon/lib/clients/snapshotter.rb

@@ -0,0 +1,43 @@
+module Euston
+  module SnapshotDaemon
+    class Snapshotter < Euston::DaemonClient
+      def initialize event_store, threshold, logger
+        @event_store = event_store
+        @threshold = threshold
+        @log = logger
+      end
+
+      private
+
+      def next_iteration
+        begin
+          stream_heads = @event_store.get_streams_to_snapshot @threshold
+          @log.debug "Found #{stream_heads.length} stream(s) eligible for snapshotting (threshold is #{@threshold})"
+
+          stream_heads.each do |stream_head|
+            pair = @event_store.get_snapshot_stream_pair stream_head.stream_id
+
+            loader = RabbitMq::ConstantLoader.new
+            loader.when(:hit => ->(klass) { take_snapshot klass, pair },
+                        :miss => ->(type) { Safely.report! "Snapshotter was unable to find a class: #{type}" })
+
+            loader.load pair.stream.committed_headers[:aggregate_type]
+          end
+        end until stopped || stream_heads.empty?
+      end
+
+      def take_snapshot klass, pair
+        instance = klass.hydrate pair.stream, pair.snapshot
+        snapshot = instance.take_snapshot
+        snapshot = EventStore::Snapshot.new pair.stream.stream_id,
+                                            pair.stream.stream_revision,
+                                            snapshot[:payload],
+                                            :version => snapshot[:version]
+
+        @log.debug "Writing snapshot: #{snapshot.inspect}"
+
+        @event_store.add_snapshot snapshot
+      end
+    end
+  end
+end

data/lib/euston-daemons/version.rb

@@ -1,5 +1,5 @@
 module Euston
   module Daemons
-    VERSION = "1.0.4"
+    VERSION = "1.1.0"
   end
 end

data/lib/euston-daemons.rb

@@ -1,6 +1,6 @@
+require 'erb-yaml'
 require 'hollywood'
 require 'logger'
-require 'require_all'
 
 if RUBY_PLATFORM.to_s == 'java'
   module Uuid
@@ -9,3 +9,8 @@ if RUBY_PLATFORM.to_s == 'java'
     end
   end
 end
+
+require 'euston-daemons/euston/daemon_environment'
+require 'euston-daemons/euston/daemon_client'
+require 'euston-daemons/euston/daemon_component'
+require 'euston-daemons/euston/daemon'

data/sample/Rakefile

@@ -0,0 +1,63 @@
+unless defined?(Bundler)
+  require 'rubygems'
+
+  begin
+    require 'bundler'
+  rescue => e
+    STDERR.puts e.message
+    STDERR.puts "Try running `bundle install`."
+    exit!
+  end
+end
+
+RAKE_ENV = (ENV['RAKE_ENV'] || 'development').to_sym
+
+Bundler.require :default, RAKE_ENV
+
+require 'euston'
+require 'euston-daemons/rake_tasks'
+
+sample_path         = File.dirname __FILE__
+pid_path            = File.join sample_path, 'pids'
+amqp_config_path    = File.join sample_path, "amqp_config.yml"
+mongoid_config_path = File.join sample_path, 'mongoid_config.yml'
+
+require 'erb-yaml'
+amqp_config = ErbYaml.read amqp_config_path, RAKE_ENV
+
+rabbitmqadmin = RabbitMqAdminCli.new
+rabbitmqadmin.initialize_vhost amqp_config[:vhost]
+rabbitmqadmin.initialize_user  amqp_config[:vhost], amqp_config[:user], amqp_config[:pass]
+
+desc "Run the command processor daemon"
+Euston::CommandProcessorRakeTask.new(RAKE_ENV) do |t|
+  require File.join sample_path, 'command_handlers'
+
+  t.amqp_config_path            = amqp_config_path
+  t.command_handler_namespaces  = ['Euston::Daemons::Sample::CommandHandlers']
+  t.daemon_config_path          = File.join sample_path, "command_processor_daemon_config.yml"
+  t.logger                      = Logger.new(STDOUT)
+  t.mongoid_config_path         = mongoid_config_path
+  t.pid_path                    = pid_path
+end
+
+desc "Run the event processor daemon"
+Euston::EventProcessorRakeTask.new(RAKE_ENV) do |t|
+  require File.join sample_path, 'event_handlers'
+
+  t.amqp_config_path            = amqp_config_path
+  t.daemon_config_path          = File.join sample_path, "event_processor_daemon_config.yml"
+  t.event_handler_namespaces    = ['Euston::Daemons::Sample::EventHandlers']
+  t.logger                      = Logger.new(STDOUT)
+  t.mongoid_config_path         = mongoid_config_path
+  t.pid_path                    = pid_path
+end
+
+desc "Run the message buffer daemon"
+Euston::MessageBufferRakeTask.new(RAKE_ENV) do |t|
+  t.amqp_config_path            = amqp_config_path
+  t.daemon_config_path          = File.join sample_path, "message_buffer_daemon_config.yml"
+  t.logger                      = Logger.new(STDOUT)
+  t.mongoid_config_path         = mongoid_config_path
+  t.pid_path                    = pid_path
+end

data/sample/amqp_config.yml

@@ -0,0 +1,14 @@
+defaults: &defaults
+  host: localhost
+  vhost: euston-daemons-sample
+  user: euston-daemons-sample-user
+  pass: password
+
+development:
+  <<: *defaults
+
+test:
+  <<: *defaults
+
+production:
+  <<: *defaults

data/sample/command_handlers.rb

@@ -0,0 +1,17 @@
+module Euston
+  module Daemons
+    module Sample
+      module CommandHandlers
+        class ContactCustomer
+          include Euston::CommandHandler
+          version 1
+        end
+
+        class PostGoods
+          include Euston::CommandHandler
+          version 1
+        end
+      end
+    end
+  end
+end

data/sample/command_processor_daemon_config.yml

@@ -0,0 +1,9 @@
+defaults: &defaults
+  client_instances: 3
+  mongo_db_name: euston-daemons-sample-event-store
+
+development:
+  <<: *defaults
+
+test:
+  <<: *defaults

data/sample/event_handlers.rb

@@ -0,0 +1,21 @@
+module Euston
+  module Daemons
+    module Sample
+      module EventHandlers
+        class SalesOffice
+          include Euston::EventHandler
+
+          consumes :customer_contacted, 1 do |headers, event|
+          end
+        end
+
+        class Warehouse
+          include Euston::EventHandler
+
+          consumes :goods_posted, 1 do |headers, event|
+          end
+        end
+      end
+    end
+  end
+end

data/sample/event_processor_daemon_config.yml

@@ -0,0 +1,8 @@
+defaults: &defaults
+  mongo_db_name: euston-daemons-sample-read-model
+
+development:
+  <<: *defaults
+
+test:
+  <<: *defaults

data/sample/message_buffer_daemon_config.yml

@@ -0,0 +1,8 @@
+defaults: &defaults
+  debug: false
+
+development:
+  <<: *defaults
+
+test:
+  <<: *defaults

data/sample/mongoid_config.yml

@@ -0,0 +1,13 @@
+defaults: &defaults
+  host: localhost
+  port: 27017
+  safe: true
+  use_utc: true
+  event_store_database: euston-daemons-sample-event-store
+  read_model_database: euston-daemons-sample-read-model
+
+development:
+  <<: *defaults
+
+test:
+  <<: *defaults