euston-daemons 1.0.5 → 1.2.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (69) hide show
  1. data/Gemfile +4 -1
  2. data/Rakefile +2 -3
  3. data/euston-daemons.gemspec +50 -39
  4. data/lib/euston-daemons.rb +14 -1
  5. data/lib/euston-daemons/euston/daemon.rb +99 -0
  6. data/lib/euston-daemons/euston/daemon_component.rb +25 -0
  7. data/lib/euston-daemons/euston/daemon_component_host.rb +66 -0
  8. data/lib/euston-daemons/euston/daemon_environment.rb +59 -0
  9. data/lib/euston-daemons/euston/exceptions.rb +9 -0
  10. data/lib/euston-daemons/euston/stopwatch.rb +15 -0
  11. data/lib/euston-daemons/pipeline/config/environment.rb +78 -0
  12. data/lib/euston-daemons/pipeline/lib/command_logger/component.rb +54 -0
  13. data/lib/euston-daemons/pipeline/lib/command_logger/log.rb +31 -0
  14. data/lib/euston-daemons/pipeline/lib/command_processor/component.rb +50 -0
  15. data/lib/euston-daemons/pipeline/lib/command_processor/default_commands/retry_failed_message.rb +13 -0
  16. data/lib/euston-daemons/pipeline/lib/command_processor/default_handlers/retry_failed_message.rb +34 -0
  17. data/lib/euston-daemons/pipeline/lib/command_processor/failed_message.rb +36 -0
  18. data/lib/euston-daemons/pipeline/lib/daemon.rb +85 -0
  19. data/lib/euston-daemons/pipeline/lib/event_processor/component.rb +67 -0
  20. data/lib/euston-daemons/pipeline/lib/event_processor/default_handlers/message_failure.rb +30 -0
  21. data/lib/euston-daemons/pipeline/lib/event_store_dispatcher/component.rb +68 -0
  22. data/lib/euston-daemons/pipeline/lib/message_buffer/buffer.rb +85 -0
  23. data/lib/euston-daemons/pipeline/lib/message_buffer/component.rb +59 -0
  24. data/lib/euston-daemons/pipeline/lib/snapshotter/component.rb +48 -0
  25. data/lib/euston-daemons/pipeline/rake_task.rb +49 -0
  26. data/lib/euston-daemons/rake_task.rb +63 -66
  27. data/lib/euston-daemons/rake_tasks.rb +3 -5
  28. data/lib/euston-daemons/version.rb +1 -1
  29. data/spec/daemons/command_processor_spec.rb +48 -0
  30. data/spec/daemons/event_processor_spec.rb +55 -0
  31. data/spec/daemons/message_buffer_spec.rb +106 -0
  32. data/spec/daemons/snapshotter_spec.rb +96 -0
  33. data/spec/spec_helper.rb +91 -0
  34. data/spec/support/factories/commands.rb +16 -0
  35. data/spec/support/factories/commit.rb +7 -0
  36. data/spec/support/factories/event_message.rb +12 -0
  37. data/spec/support/factories/events.rb +8 -0
  38. data/spec/support/filters.rb +13 -0
  39. data/spec/support/sample_model/commands.rb +14 -0
  40. data/spec/support/sample_model/counter.rb +36 -0
  41. data/spec/support/sample_model/counter2.rb +46 -0
  42. data/spec/support/stub_retrying_subscription.rb +9 -0
  43. metadata +131 -67
  44. data/lib/euston-daemons/command_processor_daemon/config/environment.rb +0 -25
  45. data/lib/euston-daemons/command_processor_daemon/lib/components/command_handler_component.rb +0 -56
  46. data/lib/euston-daemons/command_processor_daemon/lib/daemon.rb +0 -43
  47. data/lib/euston-daemons/command_processor_daemon/lib/settings.rb +0 -22
  48. data/lib/euston-daemons/command_processor_daemon/rake_task.rb +0 -34
  49. data/lib/euston-daemons/event_processor_daemon/config/environment.rb +0 -25
  50. data/lib/euston-daemons/event_processor_daemon/lib/components/event_handler_component.rb +0 -58
  51. data/lib/euston-daemons/event_processor_daemon/lib/daemon.rb +0 -71
  52. data/lib/euston-daemons/event_processor_daemon/lib/settings.rb +0 -26
  53. data/lib/euston-daemons/event_processor_daemon/rake_task.rb +0 -37
  54. data/lib/euston-daemons/framework/basic_component.rb +0 -33
  55. data/lib/euston-daemons/framework/channel_thread.rb +0 -22
  56. data/lib/euston-daemons/framework/component_shutdown.rb +0 -22
  57. data/lib/euston-daemons/framework/daemon.rb +0 -27
  58. data/lib/euston-daemons/framework/handler_bindings_component.rb +0 -56
  59. data/lib/euston-daemons/framework/queue.rb +0 -71
  60. data/lib/euston-daemons/message_buffer_daemon/config/environment.rb +0 -28
  61. data/lib/euston-daemons/message_buffer_daemon/lib/components/buffer_component.rb +0 -73
  62. data/lib/euston-daemons/message_buffer_daemon/lib/components/event_store_component.rb +0 -52
  63. data/lib/euston-daemons/message_buffer_daemon/lib/daemon.rb +0 -48
  64. data/lib/euston-daemons/message_buffer_daemon/lib/message_logger.rb +0 -54
  65. data/lib/euston-daemons/message_buffer_daemon/lib/publisher.rb +0 -56
  66. data/lib/euston-daemons/message_buffer_daemon/lib/read_model/message_log.rb +0 -36
  67. data/lib/euston-daemons/message_buffer_daemon/lib/settings.rb +0 -14
  68. data/lib/euston-daemons/message_buffer_daemon/lib/subscriber.rb +0 -60
  69. data/lib/euston-daemons/message_buffer_daemon/rake_task.rb +0 -30
@@ -1,25 +0,0 @@
1
- require 'jessica'
2
- require 'euston'
3
- require 'euston-eventstore'
4
- require 'euston-rabbitmq'
5
- require_rel '../lib'
6
-
7
- Safely::Strategy::Log.logger = EUSTON_LOG
8
- AMQP.settings.merge! ErbYaml.read(AMQP_CONFIG_PATH, EUSTON_ENV)
9
- Euston::CommandProcessorDaemon::Settings.configure ErbYaml.read(DAEMON_CONFIG_PATH, EUSTON_ENV)
10
-
11
- hash = ErbYaml.read(MONGOID_CONFIG_PATH, EUSTON_ENV)
12
-
13
- event_connection = Mongo::Connection.new hash[:host], hash[:port], :safe => hash[:safe] #, :logger => EUSTON_LOG
14
- read_connection = Mongo::Connection.new hash[:host], hash[:port], :safe => hash[:safe] #, :logger => EUSTON_LOG
15
-
16
- Euston::RabbitMq.event_store_mongodb = Mongo::DB.new(hash[:event_store_database], event_connection)
17
- Euston::RabbitMq.read_model_mongodb = Mongo::DB.new(hash[:read_model_database], read_connection)
18
-
19
- Euston::EventStore::Persistence::Mongodb::Config.instance.logger = EUSTON_LOG
20
- Euston::EventStore::Persistence::Mongodb::Config.instance.database = Euston::CommandProcessorDaemon::Settings.mongo_db_name
21
-
22
- Euston::RabbitMq.event_store = Euston::EventStore::Persistence::Mongodb::MongoPersistenceFactory.build
23
- Euston::RabbitMq.event_store.init
24
-
25
- Euston::Repository.event_store = Euston::EventStore::OptimisticEventStore.new Euston::RabbitMq.event_store
@@ -1,56 +0,0 @@
1
- module Euston
2
- module CommandProcessorDaemon
3
- class CommandHandlerComponent
4
- include Euston::Daemons::ComponentShutdown
5
-
6
- attr_reader :cli_thread
7
-
8
- def initialize daemon
9
- @daemon = daemon
10
- end
11
-
12
- def start
13
- handler_finder = Euston::RabbitMq::HandlerFinder.new [Euston::CommandHandler]
14
- handler_finder.namespaces << Euston::RabbitMq::CommandHandlers
15
- handler_finder.namespaces.push(*COMMAND_HANDLER_NAMESPACES) if Object.const_defined? 'COMMAND_HANDLER_NAMESPACES'
16
- handlers = handler_finder.find
17
-
18
- binder = Euston::RabbitMq::CommandHandlerBinder.new handlers
19
- binder.ensure_bindings_exist
20
-
21
- @cli_thread = Thread.new(@daemon, handlers) do |daemon, handlers|
22
- channel_thread = Euston::Daemons::ChannelThread.new
23
- channel_thread.daemon = daemon
24
-
25
- channel_thread.run do |channel|
26
- message_received = Proc.new do |message|
27
- command_headers = CommandHeaders.from_hash(message[:headers]).freeze
28
- command_body = message[:body].freeze
29
-
30
- handler_type = command_headers.type.to_s.camelize.to_sym
31
- handler_method_name = "__version__#{command_headers.version}"
32
-
33
- handlers.find_all { |reference| reference.name == handler_type }.each do |reference|
34
- EUSTON_LOG.debug "Delivering message to: #{reference.name}.#{handler_method_name}"
35
-
36
- reference.handler.new.send handler_method_name, command_headers, command_body
37
- end
38
- end
39
-
40
- Euston::RabbitMq::RetriableSubscription.new(channel, :command_handlers)
41
- .when(:message_received => message_received)
42
- .attach_queue_hook_listeners
43
- end
44
- end
45
- end
46
-
47
- def thread_state
48
- @cli_thread.status
49
- end
50
-
51
- def stop
52
- @cli_thread[:stop] = true
53
- end
54
- end
55
- end
56
- end
@@ -1,43 +0,0 @@
1
- module Euston
2
- module CommandProcessorDaemon
3
- class Daemon
4
- include Euston::Daemon
5
-
6
- attr_reader :clients, :queue #,:ctx
7
-
8
- def initialize #(ctx)
9
- @queue = Queue.new
10
- # @ctx = ctx
11
- @clients = {}
12
-
13
- Settings.client_instances.times do |i|
14
- @clients["command_component_#{i.succ}"] = CommandHandlerComponent.new self
15
- end
16
- end
17
-
18
- def run
19
- @clients.each do |name, component|
20
- sleep 0.25
21
- EUSTON_LOG.debug "Starting component: #{name}"
22
- component.start
23
- end
24
-
25
- EUSTON_LOG.debug "Components started"
26
-
27
- @clients.each do |name, component|
28
- EUSTON_LOG.debug("Thread state of #{name}: #{component.thread_state}")
29
- end
30
-
31
- @queue.pop #<-------- stops here until interrupted
32
-
33
- @clients.each do |name, component|
34
- EUSTON_LOG.debug "Stopping component: #{name}"
35
- component.stop
36
- end
37
-
38
- report_shutdown_reasons
39
- EUSTON_LOG.debug "Components stopped"
40
- end
41
- end
42
- end
43
- end
@@ -1,22 +0,0 @@
1
- module Euston
2
- module CommandProcessorDaemon
3
- module Settings
4
- def self.configure(cfg=nil)
5
- @config ||= {}
6
- @config.merge!(cfg) if cfg && cfg.is_a?(Hash)
7
- end
8
-
9
- def self.client_instances
10
- @config[:client_instances] || 1
11
- end
12
-
13
- def self.mongo_db_name
14
- @config[:mongo_db_name]
15
- end
16
-
17
- def self.debug
18
- @config[:debug]
19
- end
20
- end
21
- end
22
- end
@@ -1,34 +0,0 @@
1
- module Euston
2
- class CommandProcessorRakeTask < Euston::Daemons::RakeTask
3
- attr_accessor :amqp_config_path, :command_handler_namespaces, :daemon_config_path, :mongoid_config_path
4
-
5
- def initialize
6
- @command_handler_namespaces = []
7
- super(:command_processor_daemon)
8
- end
9
-
10
- def before_creating_task
11
- @daemon_path = File.expand_path(File.dirname __FILE__) + File::SEPARATOR
12
- @daemon_class = 'Euston::CommandProcessorDaemon::Daemon'
13
- end
14
-
15
- def initialize_paths
16
- EUSTON_LOG.debug "AMQP config path: #{@amqp_config_path}"
17
- Object.const_set :AMQP_CONFIG_PATH, @amqp_config_path
18
-
19
- EUSTON_LOG.debug "Command handler namespaces: #{@command_handler_namespaces}"
20
- Object.const_set :COMMAND_HANDLER_NAMESPACES, @command_handler_namespaces
21
-
22
- EUSTON_LOG.debug "Daemon config path: #{@daemon_config_path}"
23
- Object.const_set :DAEMON_CONFIG_PATH, @daemon_config_path
24
-
25
- EUSTON_LOG.debug "Mongoid config path: #{@mongoid_config_path}"
26
- Object.const_set :MONGOID_CONFIG_PATH, @mongoid_config_path
27
- end
28
-
29
- def load_environment
30
- EUSTON_LOG.debug "Loading environment"
31
- require_rel 'config/environment.rb'
32
- end
33
- end
34
- end
@@ -1,25 +0,0 @@
1
- require 'jessica'
2
- require 'euston'
3
- require 'euston-eventstore'
4
- require 'euston-rabbitmq'
5
-
6
- require_rel '../lib'
7
-
8
- Safely::Strategy::Log.logger = EUSTON_LOG
9
- AMQP.settings.merge! ErbYaml.read(AMQP_CONFIG_PATH, EUSTON_ENV)
10
- Euston::EventProcessorDaemon::Settings.configure ErbYaml.read(DAEMON_CONFIG_PATH, EUSTON_ENV)
11
-
12
- hash = ErbYaml.read(MONGOID_CONFIG_PATH, EUSTON_ENV)
13
-
14
- event_connection = Mongo::Connection.new hash[:host], hash[:port], :safe => hash[:safe] #, :logger => EUSTON_LOG
15
- read_connection = Mongo::Connection.new hash[:host], hash[:port], :safe => hash[:safe] #, :logger => EUSTON_LOG
16
-
17
- Euston::RabbitMq.event_store_mongodb = Mongo::DB.new(hash[:event_store_database], event_connection)
18
- Euston::RabbitMq.read_model_mongodb = Mongo::DB.new(hash[:read_model_database], read_connection)
19
-
20
- Mongoid.configure do |config|
21
- config.master = Euston::RabbitMq.read_model_mongodb
22
- config.logger = EUSTON_LOG
23
- end
24
-
25
- I18n.load_path += Dir[I18N_LOCALES_PATH]
@@ -1,58 +0,0 @@
1
- module Euston
2
- module EventProcessorDaemon
3
- class EventHandlerComponent
4
- include Euston::Daemons::ComponentShutdown
5
-
6
- attr_reader :cli_threads
7
-
8
- def initialize daemon
9
- @daemon = daemon
10
- @cli_threads = {}
11
- end
12
-
13
- def start
14
- handler_finder = Euston::RabbitMq::HandlerFinder.new [Euston::EventHandler]
15
- handler_finder.namespaces << Euston::RabbitMq::EventHandlers
16
- handler_finder.namespaces.push(*EVENT_HANDLER_NAMESPACES) if Object.const_defined? 'EVENT_HANDLER_NAMESPACES'
17
- handlers = handler_finder.find
18
-
19
- binder = Euston::RabbitMq::EventHandlerBinder.new handlers
20
- binder.ensure_bindings_exist
21
-
22
- handlers.each do |reference|
23
- @cli_threads[reference.handler.to_s] = Thread.new(@daemon, reference) do |daemon, reference|
24
- channel_thread = Euston::Daemons::ChannelThread.new
25
- channel_thread.daemon = daemon
26
-
27
- channel_thread.run do |channel|
28
- message_received = Proc.new do |message|
29
- event_headers = EventHeaders.from_hash message[:headers]
30
- event_body = message[:body]
31
-
32
- handler_method_name = "__event_handler__#{event_headers.type}__#{event_headers.version}"
33
-
34
- EUSTON_LOG.debug "Delivering message to: #{reference.handler}.#{handler_method_name}"
35
-
36
- reference.handler.new.send handler_method_name, event_headers.freeze, event_body.freeze
37
- end
38
-
39
- Euston::RabbitMq::RetriableSubscription.new(channel, reference.name.to_s.underscore)
40
- .when(:message_received => message_received)
41
- .attach_queue_hook_listeners
42
- end
43
- end
44
- end
45
- end
46
-
47
- def thread_state
48
- out = {}
49
- @cli_threads.each { |k, th| out[k] = th.status }
50
- out.inspect
51
- end
52
-
53
- def stop
54
- @cli_threads.values.each { |th| th[:stop] = true }
55
- end
56
- end
57
- end
58
- end
@@ -1,71 +0,0 @@
1
- module Euston
2
- module EventProcessorDaemon
3
- class Daemon
4
- include Euston::Daemon
5
-
6
- #attr_reader :ctx,:servers,:bus,:queue,:event_handler
7
- attr_reader :queue, :event_handler
8
-
9
- def initialize() #ctx)
10
- # @servers = []
11
- # @ctx = ctx
12
- # @queue = Queue.new
13
-
14
- # bus_client_port = ReadDaemonSettings.bus_port_base
15
- # bus_server_port = bus_client_port + 10
16
-
17
- # BusComponentSettings.client_side_bind_address("tcp://*:#{bus_client_port}")
18
- # BusComponentSettings.server_side_bind_address("tcp://*:#{bus_server_port}")
19
- # ReadModelComponentSettings.bus_server_side_address("tcp://localhost:#{bus_server_port}")
20
-
21
- # @bus = BusComponent.new(@ctx)
22
-
23
- # ReadDaemonSettings.server_instances.times do
24
- # @servers << ReadModelComponent.new(@ctx)
25
- # end
26
-
27
- @queue = Queue.new
28
- @event_handler = EventHandlerComponent.new self
29
- end
30
-
31
- def run
32
- # @bus.daemon = self
33
- # @bus.start
34
-
35
- # @servers.each do |ele|
36
- # sleep(0.5)
37
- # ele.daemon = self
38
- # ele.start
39
- # end
40
-
41
- name = 'event_handler'
42
- EUSTON_LOG.debug "Starting component: #{name}"
43
- @event_handler.start
44
-
45
- EUSTON_LOG.debug "Components started"
46
- EUSTON_LOG.debug "Thread state of #{name}: #{@event_handler.thread_state}"
47
-
48
- # EUSTON_LOG.debug @bus.thread_state.inspect
49
-
50
- # @servers.each do |svr|
51
- # EUSTON_LOG.debug "Server thread status: #{svr.thread_state}"
52
- # end
53
-
54
- @queue.pop #<-------- stops here until interrupted
55
-
56
- #exiting
57
- # @servers.each do |ele|
58
- # ele.stop
59
- # end
60
- # sleep(0.6)
61
- # @bus.stop
62
-
63
- EUSTON_LOG.debug "Stopping component: #{name}"
64
- @event_handler.stop
65
-
66
- report_shutdown_reasons
67
- EUSTON_LOG.debug "Components stopped"
68
- end
69
- end
70
- end
71
- end
@@ -1,26 +0,0 @@
1
- module Euston
2
- module EventProcessorDaemon
3
- module Settings
4
- def self.configure(cfg=nil)
5
- @config ||= {}
6
- @config.merge!(cfg) if cfg && cfg.is_a?(Hash)
7
- end
8
-
9
- def self.server_instances
10
- @config[:server_instances] || 2
11
- end
12
-
13
- def self.bus_port_base
14
- (@config[:zmq_base_port] || 8200).to_i
15
- end
16
-
17
- def self.mongo_db_name
18
- @config[:mongo_db_name]
19
- end
20
-
21
- def self.debug
22
- @config[:debug]
23
- end
24
- end
25
- end
26
- end
@@ -1,37 +0,0 @@
1
- module Euston
2
- class EventProcessorRakeTask < Euston::Daemons::RakeTask
3
- attr_accessor :amqp_config_path, :daemon_config_path, :event_handler_namespaces, :i18n_locales_path, :mongoid_config_path
4
-
5
- def initialize
6
- @event_handler_namespaces = []
7
- super(:event_processor_daemon)
8
- end
9
-
10
- def before_creating_task
11
- @daemon_path = File.expand_path(File.dirname __FILE__) + File::SEPARATOR
12
- @daemon_class = 'Euston::EventProcessorDaemon::Daemon'
13
- end
14
-
15
- def initialize_paths
16
- EUSTON_LOG.debug "AMQP config path: #{@amqp_config_path}"
17
- Object.const_set :AMQP_CONFIG_PATH, @amqp_config_path
18
-
19
- EUSTON_LOG.debug "Daemon config path: #{@daemon_config_path}"
20
- Object.const_set :DAEMON_CONFIG_PATH, @daemon_config_path
21
-
22
- EUSTON_LOG.debug "Event handler namespaces: #{@event_handler_namespaces}"
23
- Object.const_set :EVENT_HANDLER_NAMESPACES, @event_handler_namespaces
24
-
25
- EUSTON_LOG.debug "i18n locales path: #{@i18n_locales_path}"
26
- Object.const_set :I18N_LOCALES_PATH, @i18n_locales_path
27
-
28
- EUSTON_LOG.debug "Mongoid config path: #{@mongoid_config_path}"
29
- Object.const_set :MONGOID_CONFIG_PATH, @mongoid_config_path
30
- end
31
-
32
- def load_environment
33
- EUSTON_LOG.debug "Loading environment"
34
- require_rel 'config/environment.rb'
35
- end
36
- end
37
- end
@@ -1,33 +0,0 @@
1
- module Euston
2
- module Daemons
3
- class BasicComponent
4
- include Euston::Daemons::ComponentShutdown
5
-
6
- attr_reader :client, :cli_thread
7
-
8
- def initialize(hosted_object)
9
- @client = hosted_object
10
- end
11
-
12
- def start
13
- @cli_thread = Thread.new do
14
- begin
15
- @client.start
16
- rescue => e
17
- Thread.current[:exception] = e
18
- end
19
- check_exception_and_shutdown
20
- end
21
- end
22
-
23
- def thread_state
24
- @cli_thread.status
25
- end
26
-
27
- def stop
28
- @cli_thread[:stop] = true
29
- @client.disconnect if @client.respond_to?(:disconnect)
30
- end
31
- end
32
- end
33
- end