euston-daemons 1.0.5 → 1.2.1

data/Gemfile

@@ -1,4 +1,7 @@
 source :rubygems
 gemspec
 
-gem 'mongoid', :git => 'https://github.com/guyboertje/mongoid.git'
+gem 'cranky',             :git => 'https://github.com/leemhenson/cranky.git',           :branch => 'hashes'
+gem 'ffaker',             :git => 'https://github.com/leemhenson/ffaker.git',           :branch => 'countries'
+gem 'safely',             :git => 'https://github.com/leemhenson/safely.git',           :branch => 'override_strategies'
+gem 'hot_bunnies',        :git => 'https://github.com/ruby-amqp/hot_bunnies.git'

data/Rakefile

@@ -71,11 +71,10 @@ end
 #
 #############################################################################
 
-default_rspec_opts = %w[--colour --format Fuubar]
-
 desc "Run all examples"
 RSpec::Core::RakeTask.new(:spec) do |t|
-  t.rspec_opts = default_rspec_opts
+  t.rspec_opts = ['-r ./spec/spec_helper.rb', '--colour', '--format Fuubar']
+  t.pattern = 'spec/**/*_spec.rb'
 end
 
 #############################################################################

data/euston-daemons.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name        = 'euston-daemons'
-  s.version     = '1.0.5'
+  s.version     = '1.2.1'
   s.platform    = RUBY_PLATFORM.to_s == 'java' ? 'java' : Gem::Platform::RUBY
   s.authors     = ['Lee Henson', 'Guy Boertje']
   s.email       = ['lee.m.henson@gmail.com', 'guyboertje@gmail.com']
@@ -15,59 +15,70 @@ Gem::Specification.new do |s|
     Rakefile
     euston-daemons.gemspec
     lib/euston-daemons.rb
-    lib/euston-daemons/command_processor_daemon/config/environment.rb
-    lib/euston-daemons/command_processor_daemon/lib/components/command_handler_component.rb
-    lib/euston-daemons/command_processor_daemon/lib/daemon.rb
-    lib/euston-daemons/command_processor_daemon/lib/settings.rb
-    lib/euston-daemons/command_processor_daemon/rake_task.rb
-    lib/euston-daemons/event_processor_daemon/config/environment.rb
-    lib/euston-daemons/event_processor_daemon/lib/components/event_handler_component.rb
-    lib/euston-daemons/event_processor_daemon/lib/daemon.rb
-    lib/euston-daemons/event_processor_daemon/lib/settings.rb
-    lib/euston-daemons/event_processor_daemon/rake_task.rb
-    lib/euston-daemons/framework/basic_component.rb
-    lib/euston-daemons/framework/channel_thread.rb
-    lib/euston-daemons/framework/component_shutdown.rb
-    lib/euston-daemons/framework/daemon.rb
-    lib/euston-daemons/framework/handler_bindings_component.rb
-    lib/euston-daemons/framework/queue.rb
-    lib/euston-daemons/message_buffer_daemon/config/environment.rb
-    lib/euston-daemons/message_buffer_daemon/lib/components/buffer_component.rb
-    lib/euston-daemons/message_buffer_daemon/lib/components/event_store_component.rb
-    lib/euston-daemons/message_buffer_daemon/lib/daemon.rb
-    lib/euston-daemons/message_buffer_daemon/lib/message_logger.rb
-    lib/euston-daemons/message_buffer_daemon/lib/publisher.rb
-    lib/euston-daemons/message_buffer_daemon/lib/read_model/message_log.rb
-    lib/euston-daemons/message_buffer_daemon/lib/settings.rb
-    lib/euston-daemons/message_buffer_daemon/lib/subscriber.rb
-    lib/euston-daemons/message_buffer_daemon/rake_task.rb
+    lib/euston-daemons/euston/daemon.rb
+    lib/euston-daemons/euston/daemon_component.rb
+    lib/euston-daemons/euston/daemon_component_host.rb
+    lib/euston-daemons/euston/daemon_environment.rb
+    lib/euston-daemons/euston/exceptions.rb
+    lib/euston-daemons/euston/stopwatch.rb
+    lib/euston-daemons/pipeline/config/environment.rb
+    lib/euston-daemons/pipeline/lib/command_logger/component.rb
+    lib/euston-daemons/pipeline/lib/command_logger/log.rb
+    lib/euston-daemons/pipeline/lib/command_processor/component.rb
+    lib/euston-daemons/pipeline/lib/command_processor/default_commands/retry_failed_message.rb
+    lib/euston-daemons/pipeline/lib/command_processor/default_handlers/retry_failed_message.rb
+    lib/euston-daemons/pipeline/lib/command_processor/failed_message.rb
+    lib/euston-daemons/pipeline/lib/daemon.rb
+    lib/euston-daemons/pipeline/lib/event_processor/component.rb
+    lib/euston-daemons/pipeline/lib/event_processor/default_handlers/message_failure.rb
+    lib/euston-daemons/pipeline/lib/event_store_dispatcher/component.rb
+    lib/euston-daemons/pipeline/lib/message_buffer/buffer.rb
+    lib/euston-daemons/pipeline/lib/message_buffer/component.rb
+    lib/euston-daemons/pipeline/lib/snapshotter/component.rb
+    lib/euston-daemons/pipeline/rake_task.rb
     lib/euston-daemons/rake_task.rb
     lib/euston-daemons/rake_tasks.rb
     lib/euston-daemons/version.rb
+    spec/daemons/command_processor_spec.rb
+    spec/daemons/event_processor_spec.rb
+    spec/daemons/message_buffer_spec.rb
+    spec/daemons/snapshotter_spec.rb
+    spec/spec_helper.rb
+    spec/support/factories/commands.rb
+    spec/support/factories/commit.rb
+    spec/support/factories/event_message.rb
+    spec/support/factories/events.rb
+    spec/support/filters.rb
+    spec/support/sample_model/commands.rb
+    spec/support/sample_model/counter.rb
+    spec/support/sample_model/counter2.rb
+    spec/support/stub_retrying_subscription.rb
   ]
   # = MANIFEST =
 
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
 
-  s.add_dependency              'activemodel',        '~> 3.0.0'
-  s.add_dependency              'activesupport',      '~> 3.0.0'
+  s.add_dependency              'activemodel',        '>= 3.0.10'
+  s.add_dependency              'activesupport',      '>= 3.0.10'
   s.add_dependency              'erb-yaml',           '~> 1.0.0'
+  s.add_dependency              'euston',             '~> 1.2.0'
+  s.add_dependency              'euston-eventstore',  '~> 1.2.0'
+  s.add_dependency              'euston-rabbitmq',    '~> 1.2.0'
   s.add_dependency              'hollywood',          '~> 1.0.0'
-  s.add_dependency              'i18n',               '~> 0.5.0'
-  s.add_dependency              'require_all',        '~> 1.2.0'
+  s.add_dependency              'i18n',               '>= 0.5.0'
+  s.add_dependency              'macaddr',            '~> 1.4.0'
   s.add_dependency              'safely',             '~> 0.3.0'
 
   if RUBY_PLATFORM.to_s == 'java'
-    s.add_dependency            'euston-rabbitmq',    '~> 1.0.0'
-    s.add_dependency            'jessica',            '~> 1.0.0'
-    s.add_dependency            'jmongo',             '~> 1.0.0'
+    s.add_dependency            'hot_bunnies',        '~> 1.3.0.pre1'
+    s.add_dependency            'jmongo',             '~> 1.1.0'
   end
 
-  s.add_dependency              'euston',             '~> 1.0.0'
-  s.add_dependency              'euston-eventstore',  '~> 1.0.0'
-  s.add_dependency              'mongoid-glue',       '~> 1.0.0'
-  # s.add_dependency              'mongoid'
-
+  s.add_development_dependency  'awesome_print',      '~> 0.4.0'
+  s.add_development_dependency  'cranky',             '~> 0.2.0'
+  s.add_development_dependency  'ffaker',             '~> 1.8.0'
+  s.add_development_dependency  'fuubar',             '~> 0.0.0'
+  s.add_development_dependency  'rabbitmqadmin-cli',  '~> 1.0.2'
   s.add_development_dependency  'rspec',              '~> 2.6.0'
 end

data/lib/euston-daemons.rb

@@ -1,6 +1,6 @@
+require 'erb-yaml'
 require 'hollywood'
 require 'logger'
-require 'require_all'
 
 if RUBY_PLATFORM.to_s == 'java'
   module Uuid
@@ -8,4 +8,17 @@ if RUBY_PLATFORM.to_s == 'java'
       Java::JavaUtil::UUID.randomUUID().toString()
     end
   end
+
+  require 'euston-daemons/euston/stopwatch'
+  require 'euston-daemons/euston/exceptions'
+  require 'euston-daemons/euston/daemon_environment'
+  require 'euston-daemons/euston/daemon_component'
+  require 'euston-daemons/euston/daemon_component_host'
+  require 'euston-daemons/euston/daemon'
+else
+  require 'euston-daemons/pipeline/lib/command_processor/default_commands/retry_failed_message'
+  require 'euston-daemons/pipeline/lib/command_processor/failed_message'
+  require 'euston-daemons/pipeline/lib/message_buffer/buffer'
 end
+
+

data/lib/euston-daemons/euston/daemon.rb

@@ -0,0 +1,99 @@
+module Euston
+  class Daemon
+    def initialize data
+      @log = data[:logger]
+      @queue = Queue.new
+      @components = []
+      @errors = []
+    end
+
+    def run
+      Thread.abort_on_exception = true
+
+      pre_registration_setup
+      register_components
+      trap_exit_signals
+      start_components
+      wait_for_shutdown_event
+      stop_components
+      post_shutdown_cleanup
+      report_shutdown_reasons
+    end
+
+    private
+
+    def pre_registration_setup
+      # virtual
+    end
+
+    def register_component name, klass
+      component = Euston::DaemonComponentHost.new @log, name, klass, self
+      component.when(:fatal_error_in_component => method(:shutdown))
+
+      @components << component
+      component
+    end
+
+    def post_shutdown_cleanup
+      # virtual
+    end
+
+    def register_components
+      # virtual
+    end
+
+    def report_shutdown_reasons
+      sleep 0.25
+      reports = []
+
+      until @errors.empty? do
+        e = @errors.shift
+        next if e.nil?
+
+        case e
+        when NativeException
+          Safely.report! e.cause
+        else
+          Safely.report! e
+        end
+      end
+    end
+
+    def report_component_statuses
+      @components.each do |component|
+        @log.debug "Thread state of #{component.name}: #{component.thread_state}"
+      end
+    end
+
+    def start_components
+      @components.each do |component|
+        @log.debug "Starting component: #{component.name}"
+        component.start
+        sleep 0.350
+      end
+    end
+
+    def stop_components
+      @components.each do |component|
+        @log.debug "Stopping component: #{component.name}"
+        component.stop
+      end
+    end
+
+    def shutdown error
+      @errors.push error unless error.nil?
+      @queue.push "SHUTDOWN"
+    end
+
+    def trap_exit_signals
+      signals = %W(INT TERM) & Signal.list.keys
+      signals.each { |sig| sig.freeze }.freeze
+      signals.each { |sig| Signal.trap(sig) { @queue.push(sig) } }
+    end
+
+    def wait_for_shutdown_event
+      @log.debug "Awaiting shutdown event"
+      @queue.pop #<-------- stops here until interrupted
+    end
+  end
+end

data/lib/euston-daemons/euston/daemon_component.rb

@@ -0,0 +1,25 @@
+module Euston
+  class DaemonComponent
+    include Hollywood
+    include Daemons::Exceptions
+
+    def run
+      next_iteration unless stopped
+    end
+
+    def stop
+      @stopped = true
+      @channel.close unless @channel.nil? || !@channel.respond_to?(:close)
+    end
+
+    private
+
+    def next_iteration
+      # abstract
+    end
+
+    def stopped
+      @stopped ||= false
+    end
+  end
+end

data/lib/euston-daemons/euston/daemon_component_host.rb

@@ -0,0 +1,66 @@
+module Euston
+  class DaemonComponentHost
+    include Hollywood
+    include Daemons::Exceptions
+
+    def initialize logger, name, client, owner
+      @log, @name, @client, @owner = logger, name, client, owner
+    end
+
+    attr_reader :name
+    attr_writer :wait_time
+
+    def start
+      @thread = Thread.new do
+        while_thread_is_running do
+          begin
+            @client.run
+          rescue => e
+            Thread.current[:exception] = e
+          end
+
+          report_exceptions
+        end
+      end
+    end
+
+    def stop
+      @client.stop
+      @thread[:stop] = true
+    end
+
+    def thread_state
+      @thread.status
+    end
+
+    private
+
+    def report_exceptions
+      exception = Thread.current[:exception]
+      return if exception.nil? || ignorable_exception?(exception)
+
+      if exception.respond_to? :cause           # its a serious (Java?) exception
+        Thread.current[:stop] = true
+        callback :fatal_error_in_component, exception
+      else
+        Safely.report! exception
+      end
+    end
+
+    def thread_is_stopping?
+      Thread.current[:stop]
+    end
+
+    def wait_time
+      @wait_time ||= 0.2
+    end
+
+    def while_thread_is_running
+      until thread_is_stopping? do
+        yield
+
+        sleep wait_time
+      end
+    end
+  end
+end

data/lib/euston-daemons/euston/daemon_environment.rb

@@ -0,0 +1,59 @@
+module Euston
+  class DaemonEnvironment
+    def initialize data
+      @logger = data[:logger]
+    end
+
+    def setup
+      # abstract
+    end
+
+    private
+
+    def setup_amqp config
+      class << self.class
+        attr_accessor :amqp_connection
+      end
+
+      self.class.amqp_connection = HotBunnies.connect config
+    end
+
+    def setup_euston event_store_database_name
+      Euston::EventStore::Persistence::Mongodb::Config.instance.logger = @logger
+      Euston::EventStore::Persistence::Mongodb::Config.instance.database = event_store_database_name
+
+      class << self.class
+        attr_accessor :event_store
+      end
+
+      self.class.event_store = Euston::EventStore::Persistence::Mongodb::MongoPersistenceFactory.build
+      self.class.event_store.init
+
+      Euston::Repository.event_store = Euston::EventStore::OptimisticEventStore.new self.class.event_store
+    end
+
+    def setup_mongo config
+      # logger arg will not work until jmongo forwards it correctly
+      connection = Mongo::Connection.from_uri config[:uri], :logger => @logger
+
+      class << self.class
+        attr_accessor :event_store_mongodb, :projections_mongodb
+      end
+
+      self.class.event_store_mongodb = Mongo::DB.new config[:event_store_database], connection
+      self.class.projections_mongodb = Mongo::DB.new config[:projections_database], connection
+    end
+
+    def setup_safely hoptoad_key
+      safely_strategies = [ Safely::Strategy::Log ]
+
+      unless hoptoad_key.nil?
+        Safely::Strategy::Hoptoad.hoptoad_key = data[:environment]
+        safely_strategies << Safely::Strategy::Hoptoad
+      end
+
+      Safely::Strategy::Log.logger = @logger
+      Safely.configure { |config| config.strategies = safely_strategies }
+    end
+  end
+end

data/lib/euston-daemons/euston/exceptions.rb

@@ -0,0 +1,9 @@
+module Euston
+  module Daemons
+    module Exceptions
+      def ignorable_exception? exception
+        exception.to_s =~ /clean channel shutdown|clean connection shutdown/
+      end
+    end
+  end
+end

data/lib/euston-daemons/euston/stopwatch.rb

@@ -0,0 +1,15 @@
+module Euston
+  class Stopwatch
+    include Hollywood
+
+    def time decimal_places = 3, &block
+      start = Time.now.to_f
+      result = yield block
+      finish = Time.now.to_f
+      elapsed = finish - start
+      elapsed = elapsed.round(decimal_places) unless decimal_places.nil?
+      callback :finished, elapsed
+      result
+    end
+  end
+end