euston-daemons 1.0.5 → 1.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. data/Gemfile +4 -1
  2. data/Rakefile +2 -3
  3. data/euston-daemons.gemspec +50 -39
  4. data/lib/euston-daemons.rb +14 -1
  5. data/lib/euston-daemons/euston/daemon.rb +99 -0
  6. data/lib/euston-daemons/euston/daemon_component.rb +25 -0
  7. data/lib/euston-daemons/euston/daemon_component_host.rb +66 -0
  8. data/lib/euston-daemons/euston/daemon_environment.rb +59 -0
  9. data/lib/euston-daemons/euston/exceptions.rb +9 -0
  10. data/lib/euston-daemons/euston/stopwatch.rb +15 -0
  11. data/lib/euston-daemons/pipeline/config/environment.rb +78 -0
  12. data/lib/euston-daemons/pipeline/lib/command_logger/component.rb +54 -0
  13. data/lib/euston-daemons/pipeline/lib/command_logger/log.rb +31 -0
  14. data/lib/euston-daemons/pipeline/lib/command_processor/component.rb +50 -0
  15. data/lib/euston-daemons/pipeline/lib/command_processor/default_commands/retry_failed_message.rb +13 -0
  16. data/lib/euston-daemons/pipeline/lib/command_processor/default_handlers/retry_failed_message.rb +34 -0
  17. data/lib/euston-daemons/pipeline/lib/command_processor/failed_message.rb +36 -0
  18. data/lib/euston-daemons/pipeline/lib/daemon.rb +85 -0
  19. data/lib/euston-daemons/pipeline/lib/event_processor/component.rb +67 -0
  20. data/lib/euston-daemons/pipeline/lib/event_processor/default_handlers/message_failure.rb +30 -0
  21. data/lib/euston-daemons/pipeline/lib/event_store_dispatcher/component.rb +68 -0
  22. data/lib/euston-daemons/pipeline/lib/message_buffer/buffer.rb +85 -0
  23. data/lib/euston-daemons/pipeline/lib/message_buffer/component.rb +59 -0
  24. data/lib/euston-daemons/pipeline/lib/snapshotter/component.rb +48 -0
  25. data/lib/euston-daemons/pipeline/rake_task.rb +49 -0
  26. data/lib/euston-daemons/rake_task.rb +63 -66
  27. data/lib/euston-daemons/rake_tasks.rb +3 -5
  28. data/lib/euston-daemons/version.rb +1 -1
  29. data/spec/daemons/command_processor_spec.rb +48 -0
  30. data/spec/daemons/event_processor_spec.rb +55 -0
  31. data/spec/daemons/message_buffer_spec.rb +106 -0
  32. data/spec/daemons/snapshotter_spec.rb +96 -0
  33. data/spec/spec_helper.rb +91 -0
  34. data/spec/support/factories/commands.rb +16 -0
  35. data/spec/support/factories/commit.rb +7 -0
  36. data/spec/support/factories/event_message.rb +12 -0
  37. data/spec/support/factories/events.rb +8 -0
  38. data/spec/support/filters.rb +13 -0
  39. data/spec/support/sample_model/commands.rb +14 -0
  40. data/spec/support/sample_model/counter.rb +36 -0
  41. data/spec/support/sample_model/counter2.rb +46 -0
  42. data/spec/support/stub_retrying_subscription.rb +9 -0
  43. metadata +131 -67
  44. data/lib/euston-daemons/command_processor_daemon/config/environment.rb +0 -25
  45. data/lib/euston-daemons/command_processor_daemon/lib/components/command_handler_component.rb +0 -56
  46. data/lib/euston-daemons/command_processor_daemon/lib/daemon.rb +0 -43
  47. data/lib/euston-daemons/command_processor_daemon/lib/settings.rb +0 -22
  48. data/lib/euston-daemons/command_processor_daemon/rake_task.rb +0 -34
  49. data/lib/euston-daemons/event_processor_daemon/config/environment.rb +0 -25
  50. data/lib/euston-daemons/event_processor_daemon/lib/components/event_handler_component.rb +0 -58
  51. data/lib/euston-daemons/event_processor_daemon/lib/daemon.rb +0 -71
  52. data/lib/euston-daemons/event_processor_daemon/lib/settings.rb +0 -26
  53. data/lib/euston-daemons/event_processor_daemon/rake_task.rb +0 -37
  54. data/lib/euston-daemons/framework/basic_component.rb +0 -33
  55. data/lib/euston-daemons/framework/channel_thread.rb +0 -22
  56. data/lib/euston-daemons/framework/component_shutdown.rb +0 -22
  57. data/lib/euston-daemons/framework/daemon.rb +0 -27
  58. data/lib/euston-daemons/framework/handler_bindings_component.rb +0 -56
  59. data/lib/euston-daemons/framework/queue.rb +0 -71
  60. data/lib/euston-daemons/message_buffer_daemon/config/environment.rb +0 -28
  61. data/lib/euston-daemons/message_buffer_daemon/lib/components/buffer_component.rb +0 -73
  62. data/lib/euston-daemons/message_buffer_daemon/lib/components/event_store_component.rb +0 -52
  63. data/lib/euston-daemons/message_buffer_daemon/lib/daemon.rb +0 -48
  64. data/lib/euston-daemons/message_buffer_daemon/lib/message_logger.rb +0 -54
  65. data/lib/euston-daemons/message_buffer_daemon/lib/publisher.rb +0 -56
  66. data/lib/euston-daemons/message_buffer_daemon/lib/read_model/message_log.rb +0 -36
  67. data/lib/euston-daemons/message_buffer_daemon/lib/settings.rb +0 -14
  68. data/lib/euston-daemons/message_buffer_daemon/lib/subscriber.rb +0 -60
  69. data/lib/euston-daemons/message_buffer_daemon/rake_task.rb +0 -30
@@ -1,25 +0,0 @@
1
- require 'jessica'
2
- require 'euston'
3
- require 'euston-eventstore'
4
- require 'euston-rabbitmq'
5
- require_rel '../lib'
6
-
7
- Safely::Strategy::Log.logger = EUSTON_LOG
8
- AMQP.settings.merge! ErbYaml.read(AMQP_CONFIG_PATH, EUSTON_ENV)
9
- Euston::CommandProcessorDaemon::Settings.configure ErbYaml.read(DAEMON_CONFIG_PATH, EUSTON_ENV)
10
-
11
- hash = ErbYaml.read(MONGOID_CONFIG_PATH, EUSTON_ENV)
12
-
13
- event_connection = Mongo::Connection.new hash[:host], hash[:port], :safe => hash[:safe] #, :logger => EUSTON_LOG
14
- read_connection = Mongo::Connection.new hash[:host], hash[:port], :safe => hash[:safe] #, :logger => EUSTON_LOG
15
-
16
- Euston::RabbitMq.event_store_mongodb = Mongo::DB.new(hash[:event_store_database], event_connection)
17
- Euston::RabbitMq.read_model_mongodb = Mongo::DB.new(hash[:read_model_database], read_connection)
18
-
19
- Euston::EventStore::Persistence::Mongodb::Config.instance.logger = EUSTON_LOG
20
- Euston::EventStore::Persistence::Mongodb::Config.instance.database = Euston::CommandProcessorDaemon::Settings.mongo_db_name
21
-
22
- Euston::RabbitMq.event_store = Euston::EventStore::Persistence::Mongodb::MongoPersistenceFactory.build
23
- Euston::RabbitMq.event_store.init
24
-
25
- Euston::Repository.event_store = Euston::EventStore::OptimisticEventStore.new Euston::RabbitMq.event_store
@@ -1,56 +0,0 @@
1
- module Euston
2
- module CommandProcessorDaemon
3
- class CommandHandlerComponent
4
- include Euston::Daemons::ComponentShutdown
5
-
6
- attr_reader :cli_thread
7
-
8
- def initialize daemon
9
- @daemon = daemon
10
- end
11
-
12
- def start
13
- handler_finder = Euston::RabbitMq::HandlerFinder.new [Euston::CommandHandler]
14
- handler_finder.namespaces << Euston::RabbitMq::CommandHandlers
15
- handler_finder.namespaces.push(*COMMAND_HANDLER_NAMESPACES) if Object.const_defined? 'COMMAND_HANDLER_NAMESPACES'
16
- handlers = handler_finder.find
17
-
18
- binder = Euston::RabbitMq::CommandHandlerBinder.new handlers
19
- binder.ensure_bindings_exist
20
-
21
- @cli_thread = Thread.new(@daemon, handlers) do |daemon, handlers|
22
- channel_thread = Euston::Daemons::ChannelThread.new
23
- channel_thread.daemon = daemon
24
-
25
- channel_thread.run do |channel|
26
- message_received = Proc.new do |message|
27
- command_headers = CommandHeaders.from_hash(message[:headers]).freeze
28
- command_body = message[:body].freeze
29
-
30
- handler_type = command_headers.type.to_s.camelize.to_sym
31
- handler_method_name = "__version__#{command_headers.version}"
32
-
33
- handlers.find_all { |reference| reference.name == handler_type }.each do |reference|
34
- EUSTON_LOG.debug "Delivering message to: #{reference.name}.#{handler_method_name}"
35
-
36
- reference.handler.new.send handler_method_name, command_headers, command_body
37
- end
38
- end
39
-
40
- Euston::RabbitMq::RetriableSubscription.new(channel, :command_handlers)
41
- .when(:message_received => message_received)
42
- .attach_queue_hook_listeners
43
- end
44
- end
45
- end
46
-
47
- def thread_state
48
- @cli_thread.status
49
- end
50
-
51
- def stop
52
- @cli_thread[:stop] = true
53
- end
54
- end
55
- end
56
- end
@@ -1,43 +0,0 @@
1
- module Euston
2
- module CommandProcessorDaemon
3
- class Daemon
4
- include Euston::Daemon
5
-
6
- attr_reader :clients, :queue #,:ctx
7
-
8
- def initialize #(ctx)
9
- @queue = Queue.new
10
- # @ctx = ctx
11
- @clients = {}
12
-
13
- Settings.client_instances.times do |i|
14
- @clients["command_component_#{i.succ}"] = CommandHandlerComponent.new self
15
- end
16
- end
17
-
18
- def run
19
- @clients.each do |name, component|
20
- sleep 0.25
21
- EUSTON_LOG.debug "Starting component: #{name}"
22
- component.start
23
- end
24
-
25
- EUSTON_LOG.debug "Components started"
26
-
27
- @clients.each do |name, component|
28
- EUSTON_LOG.debug("Thread state of #{name}: #{component.thread_state}")
29
- end
30
-
31
- @queue.pop #<-------- stops here until interrupted
32
-
33
- @clients.each do |name, component|
34
- EUSTON_LOG.debug "Stopping component: #{name}"
35
- component.stop
36
- end
37
-
38
- report_shutdown_reasons
39
- EUSTON_LOG.debug "Components stopped"
40
- end
41
- end
42
- end
43
- end
@@ -1,22 +0,0 @@
1
- module Euston
2
- module CommandProcessorDaemon
3
- module Settings
4
- def self.configure(cfg=nil)
5
- @config ||= {}
6
- @config.merge!(cfg) if cfg && cfg.is_a?(Hash)
7
- end
8
-
9
- def self.client_instances
10
- @config[:client_instances] || 1
11
- end
12
-
13
- def self.mongo_db_name
14
- @config[:mongo_db_name]
15
- end
16
-
17
- def self.debug
18
- @config[:debug]
19
- end
20
- end
21
- end
22
- end
@@ -1,34 +0,0 @@
1
- module Euston
2
- class CommandProcessorRakeTask < Euston::Daemons::RakeTask
3
- attr_accessor :amqp_config_path, :command_handler_namespaces, :daemon_config_path, :mongoid_config_path
4
-
5
- def initialize
6
- @command_handler_namespaces = []
7
- super(:command_processor_daemon)
8
- end
9
-
10
- def before_creating_task
11
- @daemon_path = File.expand_path(File.dirname __FILE__) + File::SEPARATOR
12
- @daemon_class = 'Euston::CommandProcessorDaemon::Daemon'
13
- end
14
-
15
- def initialize_paths
16
- EUSTON_LOG.debug "AMQP config path: #{@amqp_config_path}"
17
- Object.const_set :AMQP_CONFIG_PATH, @amqp_config_path
18
-
19
- EUSTON_LOG.debug "Command handler namespaces: #{@command_handler_namespaces}"
20
- Object.const_set :COMMAND_HANDLER_NAMESPACES, @command_handler_namespaces
21
-
22
- EUSTON_LOG.debug "Daemon config path: #{@daemon_config_path}"
23
- Object.const_set :DAEMON_CONFIG_PATH, @daemon_config_path
24
-
25
- EUSTON_LOG.debug "Mongoid config path: #{@mongoid_config_path}"
26
- Object.const_set :MONGOID_CONFIG_PATH, @mongoid_config_path
27
- end
28
-
29
- def load_environment
30
- EUSTON_LOG.debug "Loading environment"
31
- require_rel 'config/environment.rb'
32
- end
33
- end
34
- end
@@ -1,25 +0,0 @@
1
- require 'jessica'
2
- require 'euston'
3
- require 'euston-eventstore'
4
- require 'euston-rabbitmq'
5
-
6
- require_rel '../lib'
7
-
8
- Safely::Strategy::Log.logger = EUSTON_LOG
9
- AMQP.settings.merge! ErbYaml.read(AMQP_CONFIG_PATH, EUSTON_ENV)
10
- Euston::EventProcessorDaemon::Settings.configure ErbYaml.read(DAEMON_CONFIG_PATH, EUSTON_ENV)
11
-
12
- hash = ErbYaml.read(MONGOID_CONFIG_PATH, EUSTON_ENV)
13
-
14
- event_connection = Mongo::Connection.new hash[:host], hash[:port], :safe => hash[:safe] #, :logger => EUSTON_LOG
15
- read_connection = Mongo::Connection.new hash[:host], hash[:port], :safe => hash[:safe] #, :logger => EUSTON_LOG
16
-
17
- Euston::RabbitMq.event_store_mongodb = Mongo::DB.new(hash[:event_store_database], event_connection)
18
- Euston::RabbitMq.read_model_mongodb = Mongo::DB.new(hash[:read_model_database], read_connection)
19
-
20
- Mongoid.configure do |config|
21
- config.master = Euston::RabbitMq.read_model_mongodb
22
- config.logger = EUSTON_LOG
23
- end
24
-
25
- I18n.load_path += Dir[I18N_LOCALES_PATH]
@@ -1,58 +0,0 @@
1
- module Euston
2
- module EventProcessorDaemon
3
- class EventHandlerComponent
4
- include Euston::Daemons::ComponentShutdown
5
-
6
- attr_reader :cli_threads
7
-
8
- def initialize daemon
9
- @daemon = daemon
10
- @cli_threads = {}
11
- end
12
-
13
- def start
14
- handler_finder = Euston::RabbitMq::HandlerFinder.new [Euston::EventHandler]
15
- handler_finder.namespaces << Euston::RabbitMq::EventHandlers
16
- handler_finder.namespaces.push(*EVENT_HANDLER_NAMESPACES) if Object.const_defined? 'EVENT_HANDLER_NAMESPACES'
17
- handlers = handler_finder.find
18
-
19
- binder = Euston::RabbitMq::EventHandlerBinder.new handlers
20
- binder.ensure_bindings_exist
21
-
22
- handlers.each do |reference|
23
- @cli_threads[reference.handler.to_s] = Thread.new(@daemon, reference) do |daemon, reference|
24
- channel_thread = Euston::Daemons::ChannelThread.new
25
- channel_thread.daemon = daemon
26
-
27
- channel_thread.run do |channel|
28
- message_received = Proc.new do |message|
29
- event_headers = EventHeaders.from_hash message[:headers]
30
- event_body = message[:body]
31
-
32
- handler_method_name = "__event_handler__#{event_headers.type}__#{event_headers.version}"
33
-
34
- EUSTON_LOG.debug "Delivering message to: #{reference.handler}.#{handler_method_name}"
35
-
36
- reference.handler.new.send handler_method_name, event_headers.freeze, event_body.freeze
37
- end
38
-
39
- Euston::RabbitMq::RetriableSubscription.new(channel, reference.name.to_s.underscore)
40
- .when(:message_received => message_received)
41
- .attach_queue_hook_listeners
42
- end
43
- end
44
- end
45
- end
46
-
47
- def thread_state
48
- out = {}
49
- @cli_threads.each { |k, th| out[k] = th.status }
50
- out.inspect
51
- end
52
-
53
- def stop
54
- @cli_threads.values.each { |th| th[:stop] = true }
55
- end
56
- end
57
- end
58
- end
@@ -1,71 +0,0 @@
1
- module Euston
2
- module EventProcessorDaemon
3
- class Daemon
4
- include Euston::Daemon
5
-
6
- #attr_reader :ctx,:servers,:bus,:queue,:event_handler
7
- attr_reader :queue, :event_handler
8
-
9
- def initialize() #ctx)
10
- # @servers = []
11
- # @ctx = ctx
12
- # @queue = Queue.new
13
-
14
- # bus_client_port = ReadDaemonSettings.bus_port_base
15
- # bus_server_port = bus_client_port + 10
16
-
17
- # BusComponentSettings.client_side_bind_address("tcp://*:#{bus_client_port}")
18
- # BusComponentSettings.server_side_bind_address("tcp://*:#{bus_server_port}")
19
- # ReadModelComponentSettings.bus_server_side_address("tcp://localhost:#{bus_server_port}")
20
-
21
- # @bus = BusComponent.new(@ctx)
22
-
23
- # ReadDaemonSettings.server_instances.times do
24
- # @servers << ReadModelComponent.new(@ctx)
25
- # end
26
-
27
- @queue = Queue.new
28
- @event_handler = EventHandlerComponent.new self
29
- end
30
-
31
- def run
32
- # @bus.daemon = self
33
- # @bus.start
34
-
35
- # @servers.each do |ele|
36
- # sleep(0.5)
37
- # ele.daemon = self
38
- # ele.start
39
- # end
40
-
41
- name = 'event_handler'
42
- EUSTON_LOG.debug "Starting component: #{name}"
43
- @event_handler.start
44
-
45
- EUSTON_LOG.debug "Components started"
46
- EUSTON_LOG.debug "Thread state of #{name}: #{@event_handler.thread_state}"
47
-
48
- # EUSTON_LOG.debug @bus.thread_state.inspect
49
-
50
- # @servers.each do |svr|
51
- # EUSTON_LOG.debug "Server thread status: #{svr.thread_state}"
52
- # end
53
-
54
- @queue.pop #<-------- stops here until interrupted
55
-
56
- #exiting
57
- # @servers.each do |ele|
58
- # ele.stop
59
- # end
60
- # sleep(0.6)
61
- # @bus.stop
62
-
63
- EUSTON_LOG.debug "Stopping component: #{name}"
64
- @event_handler.stop
65
-
66
- report_shutdown_reasons
67
- EUSTON_LOG.debug "Components stopped"
68
- end
69
- end
70
- end
71
- end
@@ -1,26 +0,0 @@
1
- module Euston
2
- module EventProcessorDaemon
3
- module Settings
4
- def self.configure(cfg=nil)
5
- @config ||= {}
6
- @config.merge!(cfg) if cfg && cfg.is_a?(Hash)
7
- end
8
-
9
- def self.server_instances
10
- @config[:server_instances] || 2
11
- end
12
-
13
- def self.bus_port_base
14
- (@config[:zmq_base_port] || 8200).to_i
15
- end
16
-
17
- def self.mongo_db_name
18
- @config[:mongo_db_name]
19
- end
20
-
21
- def self.debug
22
- @config[:debug]
23
- end
24
- end
25
- end
26
- end
@@ -1,37 +0,0 @@
1
- module Euston
2
- class EventProcessorRakeTask < Euston::Daemons::RakeTask
3
- attr_accessor :amqp_config_path, :daemon_config_path, :event_handler_namespaces, :i18n_locales_path, :mongoid_config_path
4
-
5
- def initialize
6
- @event_handler_namespaces = []
7
- super(:event_processor_daemon)
8
- end
9
-
10
- def before_creating_task
11
- @daemon_path = File.expand_path(File.dirname __FILE__) + File::SEPARATOR
12
- @daemon_class = 'Euston::EventProcessorDaemon::Daemon'
13
- end
14
-
15
- def initialize_paths
16
- EUSTON_LOG.debug "AMQP config path: #{@amqp_config_path}"
17
- Object.const_set :AMQP_CONFIG_PATH, @amqp_config_path
18
-
19
- EUSTON_LOG.debug "Daemon config path: #{@daemon_config_path}"
20
- Object.const_set :DAEMON_CONFIG_PATH, @daemon_config_path
21
-
22
- EUSTON_LOG.debug "Event handler namespaces: #{@event_handler_namespaces}"
23
- Object.const_set :EVENT_HANDLER_NAMESPACES, @event_handler_namespaces
24
-
25
- EUSTON_LOG.debug "i18n locales path: #{@i18n_locales_path}"
26
- Object.const_set :I18N_LOCALES_PATH, @i18n_locales_path
27
-
28
- EUSTON_LOG.debug "Mongoid config path: #{@mongoid_config_path}"
29
- Object.const_set :MONGOID_CONFIG_PATH, @mongoid_config_path
30
- end
31
-
32
- def load_environment
33
- EUSTON_LOG.debug "Loading environment"
34
- require_rel 'config/environment.rb'
35
- end
36
- end
37
- end
@@ -1,33 +0,0 @@
1
- module Euston
2
- module Daemons
3
- class BasicComponent
4
- include Euston::Daemons::ComponentShutdown
5
-
6
- attr_reader :client, :cli_thread
7
-
8
- def initialize(hosted_object)
9
- @client = hosted_object
10
- end
11
-
12
- def start
13
- @cli_thread = Thread.new do
14
- begin
15
- @client.start
16
- rescue => e
17
- Thread.current[:exception] = e
18
- end
19
- check_exception_and_shutdown
20
- end
21
- end
22
-
23
- def thread_state
24
- @cli_thread.status
25
- end
26
-
27
- def stop
28
- @cli_thread[:stop] = true
29
- @client.disconnect if @client.respond_to?(:disconnect)
30
- end
31
- end
32
- end
33
- end