euston-daemons 1.0.5 → 1.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. data/Gemfile +4 -1
  2. data/Rakefile +2 -3
  3. data/euston-daemons.gemspec +50 -39
  4. data/lib/euston-daemons.rb +14 -1
  5. data/lib/euston-daemons/euston/daemon.rb +99 -0
  6. data/lib/euston-daemons/euston/daemon_component.rb +25 -0
  7. data/lib/euston-daemons/euston/daemon_component_host.rb +66 -0
  8. data/lib/euston-daemons/euston/daemon_environment.rb +59 -0
  9. data/lib/euston-daemons/euston/exceptions.rb +9 -0
  10. data/lib/euston-daemons/euston/stopwatch.rb +15 -0
  11. data/lib/euston-daemons/pipeline/config/environment.rb +78 -0
  12. data/lib/euston-daemons/pipeline/lib/command_logger/component.rb +54 -0
  13. data/lib/euston-daemons/pipeline/lib/command_logger/log.rb +31 -0
  14. data/lib/euston-daemons/pipeline/lib/command_processor/component.rb +50 -0
  15. data/lib/euston-daemons/pipeline/lib/command_processor/default_commands/retry_failed_message.rb +13 -0
  16. data/lib/euston-daemons/pipeline/lib/command_processor/default_handlers/retry_failed_message.rb +34 -0
  17. data/lib/euston-daemons/pipeline/lib/command_processor/failed_message.rb +36 -0
  18. data/lib/euston-daemons/pipeline/lib/daemon.rb +85 -0
  19. data/lib/euston-daemons/pipeline/lib/event_processor/component.rb +67 -0
  20. data/lib/euston-daemons/pipeline/lib/event_processor/default_handlers/message_failure.rb +30 -0
  21. data/lib/euston-daemons/pipeline/lib/event_store_dispatcher/component.rb +68 -0
  22. data/lib/euston-daemons/pipeline/lib/message_buffer/buffer.rb +85 -0
  23. data/lib/euston-daemons/pipeline/lib/message_buffer/component.rb +59 -0
  24. data/lib/euston-daemons/pipeline/lib/snapshotter/component.rb +48 -0
  25. data/lib/euston-daemons/pipeline/rake_task.rb +49 -0
  26. data/lib/euston-daemons/rake_task.rb +63 -66
  27. data/lib/euston-daemons/rake_tasks.rb +3 -5
  28. data/lib/euston-daemons/version.rb +1 -1
  29. data/spec/daemons/command_processor_spec.rb +48 -0
  30. data/spec/daemons/event_processor_spec.rb +55 -0
  31. data/spec/daemons/message_buffer_spec.rb +106 -0
  32. data/spec/daemons/snapshotter_spec.rb +96 -0
  33. data/spec/spec_helper.rb +91 -0
  34. data/spec/support/factories/commands.rb +16 -0
  35. data/spec/support/factories/commit.rb +7 -0
  36. data/spec/support/factories/event_message.rb +12 -0
  37. data/spec/support/factories/events.rb +8 -0
  38. data/spec/support/filters.rb +13 -0
  39. data/spec/support/sample_model/commands.rb +14 -0
  40. data/spec/support/sample_model/counter.rb +36 -0
  41. data/spec/support/sample_model/counter2.rb +46 -0
  42. data/spec/support/stub_retrying_subscription.rb +9 -0
  43. metadata +131 -67
  44. data/lib/euston-daemons/command_processor_daemon/config/environment.rb +0 -25
  45. data/lib/euston-daemons/command_processor_daemon/lib/components/command_handler_component.rb +0 -56
  46. data/lib/euston-daemons/command_processor_daemon/lib/daemon.rb +0 -43
  47. data/lib/euston-daemons/command_processor_daemon/lib/settings.rb +0 -22
  48. data/lib/euston-daemons/command_processor_daemon/rake_task.rb +0 -34
  49. data/lib/euston-daemons/event_processor_daemon/config/environment.rb +0 -25
  50. data/lib/euston-daemons/event_processor_daemon/lib/components/event_handler_component.rb +0 -58
  51. data/lib/euston-daemons/event_processor_daemon/lib/daemon.rb +0 -71
  52. data/lib/euston-daemons/event_processor_daemon/lib/settings.rb +0 -26
  53. data/lib/euston-daemons/event_processor_daemon/rake_task.rb +0 -37
  54. data/lib/euston-daemons/framework/basic_component.rb +0 -33
  55. data/lib/euston-daemons/framework/channel_thread.rb +0 -22
  56. data/lib/euston-daemons/framework/component_shutdown.rb +0 -22
  57. data/lib/euston-daemons/framework/daemon.rb +0 -27
  58. data/lib/euston-daemons/framework/handler_bindings_component.rb +0 -56
  59. data/lib/euston-daemons/framework/queue.rb +0 -71
  60. data/lib/euston-daemons/message_buffer_daemon/config/environment.rb +0 -28
  61. data/lib/euston-daemons/message_buffer_daemon/lib/components/buffer_component.rb +0 -73
  62. data/lib/euston-daemons/message_buffer_daemon/lib/components/event_store_component.rb +0 -52
  63. data/lib/euston-daemons/message_buffer_daemon/lib/daemon.rb +0 -48
  64. data/lib/euston-daemons/message_buffer_daemon/lib/message_logger.rb +0 -54
  65. data/lib/euston-daemons/message_buffer_daemon/lib/publisher.rb +0 -56
  66. data/lib/euston-daemons/message_buffer_daemon/lib/read_model/message_log.rb +0 -36
  67. data/lib/euston-daemons/message_buffer_daemon/lib/settings.rb +0 -14
  68. data/lib/euston-daemons/message_buffer_daemon/lib/subscriber.rb +0 -60
  69. data/lib/euston-daemons/message_buffer_daemon/rake_task.rb +0 -30
@@ -0,0 +1,78 @@
1
+ require 'i18n'
2
+ require 'hot_bunnies'
3
+ require 'macaddr'
4
+ require 'euston-rabbitmq'
5
+
6
+ require 'euston-daemons/pipeline/lib/command_processor/failed_message'
7
+ require 'euston-daemons/pipeline/lib/command_processor/default_handlers/retry_failed_message'
8
+ require 'euston-daemons/pipeline/lib/command_processor/component'
9
+ require 'euston-daemons/pipeline/lib/command_logger/log'
10
+ require 'euston-daemons/pipeline/lib/command_logger/component'
11
+ require 'euston-daemons/pipeline/lib/event_processor/default_handlers/message_failure'
12
+ require 'euston-daemons/pipeline/lib/event_processor/component'
13
+ require 'euston-daemons/pipeline/lib/event_store_dispatcher/component'
14
+ require 'euston-daemons/pipeline/lib/message_buffer/buffer'
15
+ require 'euston-daemons/pipeline/lib/message_buffer/component'
16
+ require 'euston-daemons/pipeline/lib/snapshotter/component'
17
+ require 'euston-daemons/pipeline/lib/daemon'
18
+
19
+ module Euston
20
+ module Daemons
21
+ module Pipeline
22
+ class DaemonEnvironment < Euston::DaemonEnvironment
23
+ class << self
24
+ attr_accessor :command_processors,
25
+ :command_handler_namespaces,
26
+ :command_loggers,
27
+ :command_log_ignores,
28
+ :event_handler_namespaces,
29
+ :event_processors,
30
+ :event_store_dispatchers,
31
+ :event_store_dispatcher_wait_time,
32
+ :message_buffers,
33
+ :message_buffer_wait_time,
34
+ :snapshotters,
35
+ :snapshotter_wait_time,
36
+ :snapshot_threshold,
37
+ :user_defined_components
38
+ end
39
+
40
+ def initialize data
41
+ @logger = data[:logger]
42
+ @daemon_config = data[:daemon_config]
43
+ @i18n_locales_path = @daemon_config[:i18n_locales_path]
44
+
45
+ @amqp_config = ErbYaml.read data[:amqp_config_path], data[:environment]
46
+ @mongo_config = ErbYaml.read data[:mongo_config_path], data[:environment]
47
+
48
+ self.class.command_handler_namespaces = data[:command_handler_namespaces]
49
+ self.class.event_handler_namespaces = data[:event_handler_namespaces]
50
+ self.class.user_defined_components = data[:user_defined_components]
51
+
52
+ self.class.command_processors = @daemon_config[:command_processors].to_i
53
+ self.class.command_loggers = @daemon_config[:command_loggers].to_i
54
+ self.class.command_log_ignores = @daemon_config[:command_log_ignores]
55
+ self.class.event_processors = @daemon_config[:event_processors].to_i
56
+ self.class.event_store_dispatchers = @daemon_config[:event_store_dispatchers].to_i
57
+ self.class.event_store_dispatcher_wait_time = @daemon_config[:event_store_dispatcher_wait_time].to_f
58
+ self.class.message_buffers = @daemon_config[:message_buffers].to_i
59
+ self.class.message_buffer_wait_time = @daemon_config[:message_buffer_wait_time].to_f
60
+ self.class.snapshotters = @daemon_config[:snapshotters].to_i
61
+ self.class.snapshotter_wait_time = @daemon_config[:snapshotter_wait_time].to_f
62
+ self.class.snapshot_threshold = @daemon_config[:snapshot_threshold].to_i
63
+ end
64
+
65
+ def setup
66
+ setup_safely @daemon_config[:hoptoad_key]
67
+ setup_amqp @amqp_config
68
+ setup_mongo @mongo_config
69
+ setup_euston @mongo_config[:event_store_database]
70
+
71
+ I18n.load_path += Dir[@i18n_locales_path] unless @i18n_locales_path.nil?
72
+
73
+ self.class
74
+ end
75
+ end
76
+ end
77
+ end
78
+ end
@@ -0,0 +1,54 @@
1
+ module Euston
2
+ module Daemons
3
+ module Pipeline
4
+ module CommandLogger
5
+ class Component < Euston::DaemonComponent
6
+ extend RabbitMq::Exchanges
7
+ extend RabbitMq::Queues
8
+
9
+ def initialize channel, id = 1, logger = Euston::NullLogger.instance
10
+ @id = id
11
+ @log = logger
12
+ @channel = channel
13
+ @channel.prefetch = 1
14
+ @command_log = Log.new DaemonEnvironment.event_store_mongodb
15
+ @exchange = self.class.get_exchange channel, :commands
16
+ @queue = self.class.get_queue channel, @command_log.name
17
+ @queue.bind @exchange, :routing_key => "#{@exchange.name}.#"
18
+
19
+ @queue.when(:message_decode_failed => method(:log_failure),
20
+ :message_failed => method(:message_failed),
21
+ :message_received => method(:write_message_to_log))
22
+
23
+ @consumer = @queue.consumer
24
+ end
25
+
26
+ private
27
+
28
+ def log_failure message, error
29
+ return if ignorable_exception? error
30
+
31
+ text = "Command logger #{@id} failed.\n [Error] #{error.message}\n [Payload] #{message}"
32
+ err = Euston::RabbitMq::MessageDecodeFailedError.new text
33
+ err.set_backtrace error.backtrace
34
+
35
+ Safely.report! err
36
+ end
37
+
38
+ def message_failed message, error, reactive_message
39
+ log_failure message, error
40
+ reactive_message.ack
41
+ end
42
+
43
+ def next_iteration
44
+ @queue.safe_subscribe_with_timeout @consumer
45
+ end
46
+
47
+ def write_message_to_log message
48
+ @command_log.write_command message unless DaemonEnvironment.command_log_ignores.include? message[:headers][:type]
49
+ end
50
+ end
51
+ end
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,31 @@
1
+ module Euston
2
+ module Daemons
3
+ module Pipeline
4
+ module CommandLogger
5
+ class Log
6
+ def initialize mongodb
7
+ @name = 'command_log'
8
+ mongodb.create_collection @name unless mongodb.collection_names.include? @name
9
+
10
+ @collection = mongodb.collection @name
11
+ end
12
+
13
+ attr_reader :name
14
+
15
+ def write_command command
16
+ timestamp = Time.now.to_f
17
+
18
+ doc = { '_id' => command[:headers][:id],
19
+ 'type' => command[:headers][:type],
20
+ 'version' => command[:headers][:version],
21
+ 'timestamp' => timestamp,
22
+ 'timestamp_for_humans' => Time.at(timestamp),
23
+ 'json' => ActiveSupport::JSON.encode(command) }
24
+
25
+ @collection.insert doc
26
+ end
27
+ end
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,50 @@
1
+ module Euston
2
+ module Daemons
3
+ module Pipeline
4
+ module CommandProcessor
5
+ class Component < Euston::DaemonComponent
6
+ include Euston::CommandHandlerPrivateMethodNames
7
+
8
+ def initialize channel, handlers, id = 1, logger = Euston::NullLogger.instance
9
+ @channel = channel
10
+ @channel.prefetch = 1
11
+ @handlers = handlers
12
+ @id = id
13
+ @log = logger
14
+ @process_method = method(:process_message)
15
+ @stopwatch = Stopwatch.new.when(:finished => method(:log_elapsed_time))
16
+ end
17
+
18
+ private
19
+
20
+ def next_iteration
21
+ Euston::RabbitMq::RetryingSubscription.new(@channel, :command_handlers, @log)
22
+ .when(:message_received => @process_method)
23
+ .subscribe
24
+ end
25
+
26
+ def process_message message
27
+ @stopwatch.time do
28
+ command_headers = CommandHeaders.from_hash(message[:headers]).freeze
29
+ command_body = message[:body].freeze
30
+
31
+ handler_type = command_headers.type.to_s.camelize.to_sym
32
+ reference = @handlers.find { |reference| reference.name == handler_type }
33
+
34
+ if reference.nil?
35
+ Euston::CommandBus.publish command_headers, command_body, @log
36
+ else
37
+ handler_method_name = self.class.command_handler_method_name command_headers.version
38
+ reference.handler.new.send handler_method_name, command_headers, command_body
39
+ end
40
+ end
41
+ end
42
+
43
+ def log_elapsed_time elapsed_time
44
+ @log.debug "Command handler client #{@id} processed a message in #{elapsed_time} sec(s)"
45
+ end
46
+ end
47
+ end
48
+ end
49
+ end
50
+ end
@@ -0,0 +1,13 @@
1
+ module Euston
2
+ module Daemons
3
+ module Pipeline
4
+ module CommandProcessor
5
+ module DefaultCommands
6
+ class RetryFailedMessage < Euston::Command
7
+ validates :id, :presence => true, :format => { :with => /^([0-9a-fA-F]){8}-([0-9a-fA-F]){4}-([0-9a-fA-F]){4}-([0-9a-fA-F]){4}-([0-9a-fA-F]){12}$/ }
8
+ end
9
+ end
10
+ end
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,34 @@
1
+ module Euston
2
+ module Daemons
3
+ module Pipeline
4
+ module CommandProcessor
5
+ module DefaultHandlers
6
+ class RetryFailedMessage
7
+ include Euston::CommandHandler
8
+
9
+ version 1 do |headers, command|
10
+ failed_messages = FailedMessage.new DaemonEnvironment.event_store_mongodb
11
+ message = failed_messages.get_by_id command[:id]
12
+
13
+ unless message.nil?
14
+ routing_key = message['routing_key']
15
+ exchange = routing_key.split('.').first
16
+
17
+ headers = message['headers'].merge(:id => command[:id],
18
+ :type => message['type'],
19
+ :version => message['version'] )
20
+
21
+ retry_message = { :headers => headers, :body => message['body'] }
22
+
23
+ buffer = MessageBuffer::Buffer.new DaemonEnvironment.event_store_mongodb
24
+ buffer.enqueue exchange, retry_message
25
+
26
+ failed_messages.remove_by_id command[:id]
27
+ end
28
+ end
29
+ end
30
+ end
31
+ end
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,36 @@
1
+ module Euston
2
+ module Daemons
3
+ module Pipeline
4
+ module CommandProcessor
5
+ class FailedMessage
6
+ def initialize mongodb
7
+ name = 'failed_messages'
8
+ mongodb.create_collection name unless mongodb.collection_names.include? name
9
+
10
+ @collection = mongodb.collection name
11
+ @collection.ensure_index [ ['failure_timestamp', Mongo::ASCENDING] ], :unique => false, :name => 'failed_messages_failure_timestamp_index'
12
+ end
13
+
14
+ def get_by_id id
15
+ @collection.find_one({ '_id' => id })
16
+ end
17
+
18
+ def find_all
19
+ @collection.find({}, { :sort => [ 'failure_timestamp', Mongo::DESCENDING ] })
20
+ end
21
+
22
+ def log_failure failure
23
+ failure.recursive_stringify_keys!
24
+ failure['_id'] = failure.delete 'message_id'
25
+
26
+ @collection.save(failure, :safe => { :fsync => true })
27
+ end
28
+
29
+ def remove_by_id id
30
+ @collection.remove({ '_id' => id }, :safe => { :fsync => true })
31
+ end
32
+ end
33
+ end
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,85 @@
1
+ module Euston
2
+ module Daemons
3
+ module Pipeline
4
+ class Daemon < Euston::Daemon
5
+ private
6
+
7
+ def env
8
+ DaemonEnvironment
9
+ end
10
+
11
+ def post_shutdown_cleanup
12
+ env.amqp_connection.close
13
+ end
14
+
15
+ def pre_registration_setup
16
+ channel = env.amqp_connection.create_channel
17
+
18
+ begin
19
+ handler_finder = RabbitMq::HandlerFinder.new [AggregateRoot, Euston::CommandHandler], @log
20
+ handler_finder.namespaces.push Pipeline::CommandProcessor::DefaultHandlers, *DaemonEnvironment.command_handler_namespaces
21
+ @command_handler_references = handler_finder.find
22
+
23
+ binder = RabbitMq::CommandHandlerBinder.new channel, @command_handler_references, @log
24
+ binder.ensure_bindings_exist
25
+
26
+ handler_finder = RabbitMq::HandlerFinder.new [Euston::EventHandler], @log
27
+ handler_finder.namespaces.push Pipeline::EventProcessor::DefaultHandlers, *DaemonEnvironment.event_handler_namespaces
28
+ @event_handler_references = handler_finder.find
29
+
30
+ binder = RabbitMq::EventHandlerBinder.new channel, @event_handler_references, @log
31
+ binder.ensure_bindings_exist
32
+ rescue NativeException => e
33
+ @log.error "Caught the following native exception:\n\n#{e}\n\nDid you forget to create your rabbitmq vhost and user for this environment?"
34
+ raise e
35
+ ensure
36
+ channel.close
37
+ end
38
+ end
39
+
40
+ def register_components
41
+ (1..env.command_loggers).each do |i|
42
+ component = CommandLogger::Component.new env.amqp_connection.create_channel, i, @log
43
+ register_component "command_logger_#{i}".to_sym, component
44
+ end
45
+
46
+ (1..env.command_processors).each do |i|
47
+ component = CommandProcessor::Component.new env.amqp_connection.create_channel, @command_handler_references, i, @log
48
+ register_component "command_processor_#{i}", component
49
+ end
50
+
51
+ (1..env.event_store_dispatchers).each do |i|
52
+ component = EventStoreDispatcher::Component.new env.amqp_connection.create_channel, "#{Mac.addr.to_s} #{i}", @log
53
+ component = register_component "event_store_dispatcher_#{i}", component
54
+ component.wait_time = env.event_store_dispatcher_wait_time
55
+ end
56
+
57
+ (1..env.message_buffers).each do |i|
58
+ component = MessageBuffer::Component.new env.amqp_connection.create_channel, "#{Mac.addr.to_s} #{i}", @log
59
+ component = register_component "message_buffer_#{i}", component
60
+ component.wait_time = env.message_buffer_wait_time
61
+ end
62
+
63
+ (1..env.snapshotters).each do |i|
64
+ component = Snapshotter::Component.new Euston::Repository.event_store, env.snapshot_threshold, i, @log
65
+ component = register_component "snapshotter_#{i}", component
66
+ component.wait_time = env.snapshotter_wait_time
67
+ end
68
+
69
+ (1..env.event_processors).each do |i|
70
+ component = EventProcessor::Component.new env.amqp_connection.create_channel, @event_handler_references, i, @log
71
+ register_component "event_processor_#{i}", component
72
+ end
73
+
74
+ env.user_defined_components.each do |udc|
75
+ udc[:quantity].times do |i|
76
+ component = udc[:factory_method].call env, i, @log
77
+ component = register_component "#{udc[:name]}_#{i}", component
78
+ component.wait_time = udc[:wait_time] || 0.2
79
+ end
80
+ end
81
+ end
82
+ end
83
+ end
84
+ end
85
+ end
@@ -0,0 +1,67 @@
1
+ module Euston
2
+ module Daemons
3
+ module Pipeline
4
+ module EventProcessor
5
+ class Component < Euston::DaemonComponent
6
+ include Euston::EventHandlerPrivateMethodNames
7
+
8
+ def initialize channel, references, id = 1, logger = Euston::NullLogger.instance
9
+ @channel = channel
10
+ @channel.prefetch = 1
11
+ @id = id
12
+ @references = references
13
+ @log = logger
14
+ @process_method = method(:process_message)
15
+ @stopwatch = Stopwatch.new.when(:finished => method(:log_elapsed_time))
16
+
17
+ @subscriptions = @references.map do |r|
18
+ { :reference => r,
19
+ :subscription => RabbitMq::RetryingSubscription.new(@channel, r.name.to_s.underscore, @log)
20
+ .when(:message_received => @process_method) }
21
+ end
22
+ end
23
+
24
+ private
25
+
26
+ def next_iteration
27
+ @subscriptions.each do |s|
28
+ @reference = s[:reference]
29
+ s[:subscription].get
30
+ end
31
+ end
32
+
33
+ def process_message message
34
+ @stopwatch.time do
35
+ event_headers = EventHeaders.from_hash message[:headers]
36
+ event_body = message[:body]
37
+
38
+ handler_is_aggregate_root = @reference.handler.include? Euston::AggregateRoot
39
+
40
+ if handler_is_aggregate_root
41
+ id_getter_method = self.class.id_from_event_method_name event_headers.type, event_headers.version
42
+ id = @reference.handler.send id_getter_method, event_body
43
+
44
+ raise "Unable to extract the aggregate root id from an #{event_headers.type} event. Did you forget to add an { :id => :field_name } argument to your subscribes block?" if id.nil?
45
+
46
+ handler_instance = Euston::Repository.find @reference.handler, id
47
+ handler_instance.log = @log
48
+ handler_instance.consume_event_subscription event_headers.freeze, event_body.freeze
49
+
50
+ Euston::Repository.save handler_instance
51
+ else
52
+ handler_method = self.class.event_handler_method_name event_headers.type, event_headers.version
53
+ handler_instance = @reference.handler.new
54
+ handler_instance.log = @log if handler_instance.respond_to? :log=
55
+ handler_instance.send handler_method, event_headers.freeze, event_body.freeze
56
+ end
57
+ end
58
+ end
59
+
60
+ def log_elapsed_time elapsed_time
61
+ @log.debug "Event handler client #{@reference.name.to_s.underscore} processed a message in #{elapsed_time} sec(s)"
62
+ end
63
+ end
64
+ end
65
+ end
66
+ end
67
+ end