euston-daemons 1.0.4-java → 1.1.0-java
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile +3 -1
- data/Rakefile +2 -3
- data/euston-daemons.gemspec +61 -26
- data/lib/euston-daemons/command_processor_daemon/config/environment.rb +27 -18
- data/lib/euston-daemons/command_processor_daemon/lib/clients/command_handler.rb +37 -0
- data/lib/euston-daemons/command_processor_daemon/lib/command_handlers/retry_failed_message.rb +35 -0
- data/lib/euston-daemons/command_processor_daemon/lib/daemon.rb +11 -32
- data/lib/euston-daemons/command_processor_daemon/lib/mongo_models/failed_message.rb +34 -0
- data/lib/euston-daemons/command_processor_daemon/rake_task.rb +15 -13
- data/lib/euston-daemons/euston/daemon.rb +93 -0
- data/lib/euston-daemons/euston/daemon_client.rb +24 -0
- data/lib/euston-daemons/euston/daemon_component.rb +65 -0
- data/lib/euston-daemons/euston/daemon_environment.rb +54 -0
- data/lib/euston-daemons/event_processor_daemon/config/environment.rb +27 -16
- data/lib/euston-daemons/event_processor_daemon/lib/clients/event_handler.rb +42 -0
- data/lib/euston-daemons/event_processor_daemon/lib/daemon.rb +13 -60
- data/lib/euston-daemons/event_processor_daemon/lib/event_handlers/message_failure.rb +27 -0
- data/lib/euston-daemons/event_processor_daemon/rake_task.rb +18 -16
- data/lib/euston-daemons/message_buffer_daemon/config/environment.rb +39 -25
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/command_buffer_cleanup.rb +9 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/command_buffer_publisher.rb +9 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/command_logger.rb +9 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/euston_exchange_accessors.rb +15 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/event_buffer_cleanup.rb +9 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/event_buffer_publisher.rb +9 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/event_logger.rb +9 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/event_store_dispatcher.rb +25 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/message_buffer_cleanup.rb +52 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/message_buffer_publisher.rb +37 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/message_logger.rb +45 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/mongo_model_accessors.rb +21 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/daemon.rb +11 -42
- data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/command_buffer.rb +11 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/command_log.rb +11 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/event_buffer.rb +11 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/event_log.rb +11 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/message_buffer.rb +57 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/{read_model → mongo_models}/message_log.rb +4 -11
- data/lib/euston-daemons/message_buffer_daemon/rake_task.rb +13 -11
- data/lib/euston-daemons/rake_task.rb +41 -65
- data/lib/euston-daemons/rake_tasks.rb +5 -5
- data/lib/euston-daemons/snapshot_daemon/lib/clients/snapshotter.rb +43 -0
- data/lib/euston-daemons/version.rb +1 -1
- data/lib/euston-daemons.rb +6 -1
- data/sample/Rakefile +63 -0
- data/sample/amqp_config.yml +14 -0
- data/sample/command_handlers.rb +17 -0
- data/sample/command_processor_daemon_config.yml +9 -0
- data/sample/event_handlers.rb +21 -0
- data/sample/event_processor_daemon_config.yml +8 -0
- data/sample/message_buffer_daemon_config.yml +8 -0
- data/sample/mongoid_config.yml +13 -0
- data/sample/pids/.placeholder +0 -0
- data/spec/daemons/command_buffer_publisher_spec.rb +110 -0
- data/spec/daemons/command_handler_spec.rb +48 -0
- data/spec/daemons/event_handler_spec.rb +55 -0
- data/spec/daemons/snapshot_client_spec.rb +98 -0
- data/spec/spec_helper.rb +77 -0
- data/spec/support/factories/commands.rb +16 -0
- data/spec/support/factories/commit.rb +7 -0
- data/spec/support/factories/event_message.rb +12 -0
- data/spec/support/factories/events.rb +8 -0
- data/spec/support/filters.rb +14 -0
- data/spec/support/sample_model/commands.rb +14 -0
- data/spec/support/sample_model/counter.rb +36 -0
- data/spec/support/sample_model/counter2.rb +46 -0
- data/spec/support/stub_retrying_subscription.rb +9 -0
- metadata +134 -67
- data/lib/euston-daemons/command_processor_daemon/lib/components/command_handler_component.rb +0 -56
- data/lib/euston-daemons/command_processor_daemon/lib/settings.rb +0 -22
- data/lib/euston-daemons/event_processor_daemon/lib/components/event_handler_component.rb +0 -58
- data/lib/euston-daemons/event_processor_daemon/lib/settings.rb +0 -26
- data/lib/euston-daemons/framework/basic_component.rb +0 -33
- data/lib/euston-daemons/framework/channel_thread.rb +0 -22
- data/lib/euston-daemons/framework/component_shutdown.rb +0 -22
- data/lib/euston-daemons/framework/daemon.rb +0 -27
- data/lib/euston-daemons/framework/handler_bindings_component.rb +0 -56
- data/lib/euston-daemons/framework/queue.rb +0 -71
- data/lib/euston-daemons/message_buffer_daemon/lib/components/buffer_component.rb +0 -73
- data/lib/euston-daemons/message_buffer_daemon/lib/components/event_store_component.rb +0 -52
- data/lib/euston-daemons/message_buffer_daemon/lib/message_logger.rb +0 -54
- data/lib/euston-daemons/message_buffer_daemon/lib/publisher.rb +0 -56
- data/lib/euston-daemons/message_buffer_daemon/lib/settings.rb +0 -14
- data/lib/euston-daemons/message_buffer_daemon/lib/subscriber.rb +0 -60
@@ -1,25 +1,36 @@
|
|
1
|
+
require 'i18n'
|
1
2
|
require 'jessica'
|
2
|
-
require 'euston'
|
3
|
-
require 'euston-eventstore'
|
4
3
|
require 'euston-rabbitmq'
|
5
4
|
|
6
|
-
|
5
|
+
require 'euston-daemons/event_processor_daemon/lib/event_handlers/message_failure'
|
6
|
+
require 'euston-daemons/event_processor_daemon/lib/clients/event_handler'
|
7
|
+
require 'euston-daemons/event_processor_daemon/lib/daemon'
|
7
8
|
|
8
|
-
|
9
|
-
|
10
|
-
Euston::
|
9
|
+
module Euston
|
10
|
+
module EventProcessorDaemon
|
11
|
+
class DaemonEnvironment < Euston::DaemonEnvironment
|
12
|
+
class << self
|
13
|
+
attr_accessor :event_handler_namespaces
|
14
|
+
end
|
11
15
|
|
12
|
-
|
16
|
+
def initialize data
|
17
|
+
@logger = data[:logger]
|
18
|
+
@amqp_config = ErbYaml.read data[:amqp_config_path], data[:environment]
|
19
|
+
@daemon_config = ErbYaml.read data[:daemon_config_path], data[:environment]
|
20
|
+
@mongoid_config = ErbYaml.read data[:mongoid_config_path], data[:environment]
|
13
21
|
|
14
|
-
|
15
|
-
|
22
|
+
@i18n_locales_path = data[:i18n_locales_path]
|
23
|
+
self.class.event_handler_namespaces = data[:event_handler_namespaces]
|
24
|
+
end
|
16
25
|
|
17
|
-
|
18
|
-
|
26
|
+
def setup
|
27
|
+
setup_safely @daemon_config[:hoptoad_key]
|
28
|
+
setup_amqp @amqp_config
|
29
|
+
setup_mongo @mongoid_config
|
30
|
+
setup_euston @mongoid_config[:event_store_database]
|
19
31
|
|
20
|
-
|
21
|
-
|
22
|
-
|
32
|
+
I18n.load_path += Dir[@i18n_locales_path] unless @i18n_locales_path.nil?
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
23
36
|
end
|
24
|
-
|
25
|
-
I18n.load_path += Dir[I18N_LOCALES_PATH]
|
@@ -0,0 +1,42 @@
|
|
1
|
+
module Euston
|
2
|
+
module EventProcessorDaemon
|
3
|
+
class EventHandler < Euston::DaemonClient
|
4
|
+
def initialize channel, reference, logger
|
5
|
+
@channel = channel
|
6
|
+
@channel.prefetch 1
|
7
|
+
@reference = reference
|
8
|
+
@log = logger
|
9
|
+
end
|
10
|
+
|
11
|
+
private
|
12
|
+
|
13
|
+
def next_iteration
|
14
|
+
message_received = Proc.new do |message|
|
15
|
+
event_headers = EventHeaders.from_hash message[:headers]
|
16
|
+
event_body = message[:body]
|
17
|
+
handler_method_name = "__event_handler__#{event_headers.type}__#{event_headers.version}"
|
18
|
+
|
19
|
+
@log.debug "Delivering message to: #{@reference.handler}.#{handler_method_name}"
|
20
|
+
|
21
|
+
handler_is_aggregate_root = @reference.handler.include? Euston::AggregateRoot
|
22
|
+
|
23
|
+
if handler_is_aggregate_root
|
24
|
+
id_getter_method = "__id_from_event_#{event_headers.type}__v#{event_headers.version}__"
|
25
|
+
id = @reference.handler.send id_getter_method, event_body
|
26
|
+
handler_instance = Euston::Repository.find @reference.handler, id
|
27
|
+
else
|
28
|
+
handler_instance = @reference.handler.new
|
29
|
+
end
|
30
|
+
|
31
|
+
handler_instance.send handler_method_name, event_headers.freeze, event_body.freeze
|
32
|
+
|
33
|
+
Euston::Repository.save handler_instance if handler_is_aggregate_root
|
34
|
+
end
|
35
|
+
|
36
|
+
Euston::RabbitMq::RetryingSubscription.new(@channel, @reference.name.to_s.underscore, @log)
|
37
|
+
.when(:message_received => message_received)
|
38
|
+
.subscribe
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -1,70 +1,23 @@
|
|
1
1
|
module Euston
|
2
2
|
module EventProcessorDaemon
|
3
|
-
class Daemon
|
4
|
-
|
3
|
+
class Daemon < Euston::Daemon
|
4
|
+
private
|
5
5
|
|
6
|
-
|
7
|
-
|
6
|
+
def pre_registration_setup
|
7
|
+
handler_finder = Euston::RabbitMq::HandlerFinder.new [Euston::EventHandler]
|
8
|
+
handler_finder.namespaces.push Euston::EventProcessorDaemon::EventHandlers, *DaemonEnvironment.event_handler_namespaces
|
9
|
+
@handlers = handler_finder.find
|
8
10
|
|
9
|
-
|
10
|
-
|
11
|
-
# @ctx = ctx
|
12
|
-
# @queue = Queue.new
|
13
|
-
|
14
|
-
# bus_client_port = ReadDaemonSettings.bus_port_base
|
15
|
-
# bus_server_port = bus_client_port + 10
|
16
|
-
|
17
|
-
# BusComponentSettings.client_side_bind_address("tcp://*:#{bus_client_port}")
|
18
|
-
# BusComponentSettings.server_side_bind_address("tcp://*:#{bus_server_port}")
|
19
|
-
# ReadModelComponentSettings.bus_server_side_address("tcp://localhost:#{bus_server_port}")
|
20
|
-
|
21
|
-
# @bus = BusComponent.new(@ctx)
|
22
|
-
|
23
|
-
# ReadDaemonSettings.server_instances.times do
|
24
|
-
# @servers << ReadModelComponent.new(@ctx)
|
25
|
-
# end
|
26
|
-
|
27
|
-
@queue = Queue.new
|
28
|
-
@event_handler = EventHandlerComponent.new self
|
11
|
+
binder = Euston::RabbitMq::EventHandlerBinder.new @handlers
|
12
|
+
binder.ensure_bindings_exist
|
29
13
|
end
|
30
14
|
|
31
|
-
def
|
32
|
-
|
33
|
-
# @bus.start
|
34
|
-
|
35
|
-
# @servers.each do |ele|
|
36
|
-
# sleep(0.5)
|
37
|
-
# ele.daemon = self
|
38
|
-
# ele.start
|
39
|
-
# end
|
40
|
-
|
41
|
-
name = 'event_handler'
|
42
|
-
EUSTON_LOG.debug "Starting component: #{name}"
|
43
|
-
@event_handler.start
|
44
|
-
|
45
|
-
EUSTON_LOG.debug "Components started"
|
46
|
-
EUSTON_LOG.debug "Thread state of #{name}: #{@event_handler.thread_state}"
|
47
|
-
|
48
|
-
# EUSTON_LOG.debug @bus.thread_state.inspect
|
49
|
-
|
50
|
-
# @servers.each do |svr|
|
51
|
-
# EUSTON_LOG.debug "Server thread status: #{svr.thread_state}"
|
52
|
-
# end
|
53
|
-
|
54
|
-
@queue.pop #<-------- stops here until interrupted
|
55
|
-
|
56
|
-
#exiting
|
57
|
-
# @servers.each do |ele|
|
58
|
-
# ele.stop
|
59
|
-
# end
|
60
|
-
# sleep(0.6)
|
61
|
-
# @bus.stop
|
62
|
-
|
63
|
-
EUSTON_LOG.debug "Stopping component: #{name}"
|
64
|
-
@event_handler.stop
|
15
|
+
def register_components
|
16
|
+
env = DaemonEnvironment
|
65
17
|
|
66
|
-
|
67
|
-
|
18
|
+
@handlers.each do |reference|
|
19
|
+
register_component "#{reference.name.to_s.underscore}_event_handler".to_sym, EventHandler.new(AMQP::Channel.new, reference, @log)
|
20
|
+
end
|
68
21
|
end
|
69
22
|
end
|
70
23
|
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module Euston
|
2
|
+
module EventProcessorDaemon
|
3
|
+
module EventHandlers
|
4
|
+
class MessageFailure
|
5
|
+
include Euston::EventHandler
|
6
|
+
|
7
|
+
subscribes :message_failed, 1 do |headers, event|
|
8
|
+
headers = event[:message][:headers].dup
|
9
|
+
failure = { :message_id => headers.delete(:id),
|
10
|
+
:type => headers.delete(:type),
|
11
|
+
:version => headers.delete(:version),
|
12
|
+
:message_timestamp => headers.delete(:timestamp),
|
13
|
+
:routing_key => event[:routing_key],
|
14
|
+
:body => event[:message][:body],
|
15
|
+
:headers => headers,
|
16
|
+
:error => event[:error],
|
17
|
+
:backtrace => event[:backtrace],
|
18
|
+
:failure_timestamp => Time.now.to_f }
|
19
|
+
|
20
|
+
model = Euston::CommandProcessorDaemon::MongoModel::FailedMessage
|
21
|
+
model = model.new DaemonEnvironment.event_store_mongodb
|
22
|
+
model.log_failure failure
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -2,9 +2,9 @@ module Euston
|
|
2
2
|
class EventProcessorRakeTask < Euston::Daemons::RakeTask
|
3
3
|
attr_accessor :amqp_config_path, :daemon_config_path, :event_handler_namespaces, :i18n_locales_path, :mongoid_config_path
|
4
4
|
|
5
|
-
def initialize
|
5
|
+
def initialize environment
|
6
6
|
@event_handler_namespaces = []
|
7
|
-
super
|
7
|
+
super environment, :event_processor_daemon
|
8
8
|
end
|
9
9
|
|
10
10
|
def before_creating_task
|
@@ -12,26 +12,28 @@ module Euston
|
|
12
12
|
@daemon_class = 'Euston::EventProcessorDaemon::Daemon'
|
13
13
|
end
|
14
14
|
|
15
|
-
def
|
16
|
-
|
17
|
-
|
15
|
+
def initialize_settings
|
16
|
+
@logger.debug "AMQP config path: #{@amqp_config_path}"
|
17
|
+
@data[:amqp_config_path] = @amqp_config_path
|
18
18
|
|
19
|
-
|
20
|
-
|
19
|
+
@logger.debug "Daemon config path: #{@daemon_config_path}"
|
20
|
+
@data[:daemon_config_path] = @daemon_config_path
|
21
21
|
|
22
|
-
|
23
|
-
|
22
|
+
@logger.debug "Event handler namespaces: #{@event_handler_namespaces}"
|
23
|
+
@data[:event_handler_namespaces] = @event_handler_namespaces
|
24
24
|
|
25
|
-
|
26
|
-
|
25
|
+
@logger.debug "i18n locales path: #{@i18n_locales_path}"
|
26
|
+
@data[:i18n_locales_path] = @i18n_locales_path
|
27
27
|
|
28
|
-
|
29
|
-
|
28
|
+
@logger.debug "Mongoid config path: #{@mongoid_config_path}"
|
29
|
+
@data[:mongoid_config_path] = @mongoid_config_path
|
30
30
|
end
|
31
31
|
|
32
32
|
def load_environment
|
33
|
-
|
34
|
-
|
33
|
+
@logger.debug "Loading environment"
|
34
|
+
require 'euston-daemons/event_processor_daemon/config/environment'
|
35
|
+
|
36
|
+
Euston::EventProcessorDaemon::DaemonEnvironment.new(@data).setup
|
35
37
|
end
|
36
38
|
end
|
37
|
-
end
|
39
|
+
end
|
@@ -1,28 +1,42 @@
|
|
1
1
|
require 'jessica'
|
2
|
-
require 'euston'
|
3
|
-
require 'euston-eventstore'
|
4
2
|
require 'euston-rabbitmq'
|
5
3
|
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
4
|
+
require 'euston-daemons/message_buffer_daemon/lib/mongo_models/message_buffer'
|
5
|
+
require 'euston-daemons/message_buffer_daemon/lib/mongo_models/message_log'
|
6
|
+
require 'euston-daemons/message_buffer_daemon/lib/mongo_models/command_buffer'
|
7
|
+
require 'euston-daemons/message_buffer_daemon/lib/mongo_models/command_log'
|
8
|
+
require 'euston-daemons/message_buffer_daemon/lib/mongo_models/event_buffer'
|
9
|
+
require 'euston-daemons/message_buffer_daemon/lib/mongo_models/event_log'
|
10
|
+
require 'euston-daemons/message_buffer_daemon/lib/clients/euston_exchange_accessors'
|
11
|
+
require 'euston-daemons/message_buffer_daemon/lib/clients/mongo_model_accessors'
|
12
|
+
require 'euston-daemons/message_buffer_daemon/lib/clients/message_buffer_cleanup'
|
13
|
+
require 'euston-daemons/message_buffer_daemon/lib/clients/message_buffer_publisher'
|
14
|
+
require 'euston-daemons/message_buffer_daemon/lib/clients/message_logger'
|
15
|
+
require 'euston-daemons/message_buffer_daemon/lib/clients/command_buffer_cleanup'
|
16
|
+
require 'euston-daemons/message_buffer_daemon/lib/clients/command_buffer_publisher'
|
17
|
+
require 'euston-daemons/message_buffer_daemon/lib/clients/command_logger'
|
18
|
+
require 'euston-daemons/message_buffer_daemon/lib/clients/event_buffer_cleanup'
|
19
|
+
require 'euston-daemons/message_buffer_daemon/lib/clients/event_buffer_publisher'
|
20
|
+
require 'euston-daemons/message_buffer_daemon/lib/clients/event_logger'
|
21
|
+
require 'euston-daemons/message_buffer_daemon/lib/clients/event_store_dispatcher'
|
22
|
+
require 'euston-daemons/message_buffer_daemon/lib/daemon'
|
23
|
+
|
24
|
+
module Euston
|
25
|
+
module MessageBufferDaemon
|
26
|
+
class DaemonEnvironment < Euston::DaemonEnvironment
|
27
|
+
def initialize data
|
28
|
+
@logger = data[:logger]
|
29
|
+
@amqp_config = ErbYaml.read data[:amqp_config_path], data[:environment]
|
30
|
+
@daemon_config = ErbYaml.read data[:daemon_config_path], data[:environment]
|
31
|
+
@mongoid_config = ErbYaml.read data[:mongoid_config_path], data[:environment]
|
32
|
+
end
|
33
|
+
|
34
|
+
def setup
|
35
|
+
setup_safely @daemon_config[:hoptoad_key]
|
36
|
+
setup_amqp @amqp_config
|
37
|
+
setup_mongo @mongoid_config
|
38
|
+
setup_euston @mongoid_config[:event_store_database]
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
module Euston
|
2
|
+
module MessageBufferDaemon
|
3
|
+
module EustonExchangeAccessors
|
4
|
+
include Euston::RabbitMq::Exchanges
|
5
|
+
|
6
|
+
def commands_exchange channel
|
7
|
+
get_exchange channel, :commands
|
8
|
+
end
|
9
|
+
|
10
|
+
def events_exchange channel
|
11
|
+
get_exchange channel, :events
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module Euston
|
2
|
+
module MessageBufferDaemon
|
3
|
+
class EventStoreDispatcher < Euston::DaemonClient
|
4
|
+
extend MongoModelAccessors
|
5
|
+
|
6
|
+
def initialize
|
7
|
+
@event_buffer = self.class.event_buffer_mongo_model
|
8
|
+
@event_store = DaemonEnvironment.event_store
|
9
|
+
end
|
10
|
+
|
11
|
+
private
|
12
|
+
|
13
|
+
def next_iteration
|
14
|
+
begin
|
15
|
+
@commits = @event_store.get_undispatched_commits
|
16
|
+
|
17
|
+
@commits.each do |commit|
|
18
|
+
commit.events.each { |event| @event_buffer.buffer_new_message event.to_hash }
|
19
|
+
@event_store.mark_commit_as_dispatched commit
|
20
|
+
end
|
21
|
+
end until stopped || @commits.empty?
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
module Euston
|
2
|
+
module MessageBufferDaemon
|
3
|
+
class MessageBufferCleanup < Euston::DaemonClient
|
4
|
+
extend EustonExchangeAccessors
|
5
|
+
extend MongoModelAccessors
|
6
|
+
include Euston::RabbitMq::Queues
|
7
|
+
|
8
|
+
def initialize channel, exchange, mongo_model
|
9
|
+
@channel = channel
|
10
|
+
@mongo_model = mongo_model
|
11
|
+
|
12
|
+
@channel.prefetch(1)
|
13
|
+
@exchange = exchange
|
14
|
+
@queue = get_queue @channel, mongo_model.name
|
15
|
+
@queue.bind @exchange, :routing_key => "#{exchange.name}.#"
|
16
|
+
|
17
|
+
@queue.when(:message_decode_failed => method(:log_failure),
|
18
|
+
:message_failed => method(:message_failed),
|
19
|
+
:message_received => method(:remove_message_from_buffer))
|
20
|
+
|
21
|
+
@consumer = @queue.consumer
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
def log_failure message, error
|
27
|
+
text = "A(n) #{@exchange.name} buffer queue subscription failed. [Error] #{error.message} [Payload] #{message}"
|
28
|
+
error = Euston::RabbitMq::MessageDecodeFailedError.new text
|
29
|
+
error.set_backtrace error.backtrace
|
30
|
+
|
31
|
+
Safely.report! error
|
32
|
+
end
|
33
|
+
|
34
|
+
def message_failed message, error, reactive_message
|
35
|
+
reactive_message.ack!
|
36
|
+
log_failure message, error
|
37
|
+
end
|
38
|
+
|
39
|
+
def next_iteration
|
40
|
+
@queue.safe_subscribe_with_timeout @consumer, subscribe_timeout
|
41
|
+
end
|
42
|
+
|
43
|
+
def remove_message_from_buffer message
|
44
|
+
@mongo_model.remove_published_message message[:headers][:id]
|
45
|
+
end
|
46
|
+
|
47
|
+
def subscribe_timeout
|
48
|
+
@subscribe_timeout ||= 2000
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
module Euston
|
2
|
+
module MessageBufferDaemon
|
3
|
+
class MessageBufferPublisher < Euston::DaemonClient
|
4
|
+
extend EustonExchangeAccessors
|
5
|
+
extend MongoModelAccessors
|
6
|
+
|
7
|
+
def initialize channel, exchange, mongo_model
|
8
|
+
super()
|
9
|
+
@channel = channel
|
10
|
+
@exchange = exchange
|
11
|
+
@mongo_model = mongo_model
|
12
|
+
end
|
13
|
+
|
14
|
+
private
|
15
|
+
|
16
|
+
def dispatch_due_messages
|
17
|
+
dispatched = 0
|
18
|
+
|
19
|
+
@mongo_model.find_due_messages.each do |message|
|
20
|
+
@mongo_model.set_next_attempt message
|
21
|
+
@exchange.publish message['json'], self.class.default_publish_options.merge(:routing_key => "#{@exchange.name}.#{message['type']}")
|
22
|
+
dispatched += 1
|
23
|
+
end
|
24
|
+
|
25
|
+
dispatched
|
26
|
+
end
|
27
|
+
|
28
|
+
def next_iteration
|
29
|
+
@messages_dispatched = 0
|
30
|
+
|
31
|
+
begin
|
32
|
+
@messages_dispatched = dispatch_due_messages
|
33
|
+
end until stopped || @messages_dispatched.zero?
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
module Euston
|
2
|
+
module MessageBufferDaemon
|
3
|
+
class MessageLogger < Euston::DaemonClient
|
4
|
+
extend EustonExchangeAccessors
|
5
|
+
extend MongoModelAccessors
|
6
|
+
include Euston::RabbitMq::Queues
|
7
|
+
|
8
|
+
def initialize channel, exchange, mongo_model
|
9
|
+
@channel = channel
|
10
|
+
@exchange = exchange
|
11
|
+
@mongo_model = mongo_model
|
12
|
+
|
13
|
+
@queue = get_queue channel, mongo_model.name
|
14
|
+
@queue.bind @exchange, :routing_key => "#{exchange.name}.#"
|
15
|
+
|
16
|
+
@queue.when(:message_decode_failed => method(:log_failure),
|
17
|
+
:message_failed => method(:message_failed),
|
18
|
+
:message_received => method(:write_message_to_log))
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
|
23
|
+
def log_failure message, error
|
24
|
+
text = "A log queue subscription failed. [Error] #{error.message} [Payload] #{message}"
|
25
|
+
err = Euston::RabbitMq::MessageDecodeFailedError.new text
|
26
|
+
err.set_backtrace error.backtrace
|
27
|
+
|
28
|
+
Safely.report! err
|
29
|
+
end
|
30
|
+
|
31
|
+
def message_failed message, error, header
|
32
|
+
log_failure message, error
|
33
|
+
header.ack
|
34
|
+
end
|
35
|
+
|
36
|
+
def next_iteration
|
37
|
+
@queue.safe_subscribe
|
38
|
+
end
|
39
|
+
|
40
|
+
def write_message_to_log message
|
41
|
+
@mongo_model.log_new_message message
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module Euston
|
2
|
+
module MessageBufferDaemon
|
3
|
+
module MongoModelAccessors
|
4
|
+
def command_buffer_mongo_model
|
5
|
+
MongoModel::CommandBuffer.new DaemonEnvironment.event_store_mongodb
|
6
|
+
end
|
7
|
+
|
8
|
+
def command_log_mongo_model
|
9
|
+
MongoModel::CommandLog.new DaemonEnvironment.event_store_mongodb
|
10
|
+
end
|
11
|
+
|
12
|
+
def event_buffer_mongo_model
|
13
|
+
MongoModel::EventBuffer.new DaemonEnvironment.event_store_mongodb
|
14
|
+
end
|
15
|
+
|
16
|
+
def event_log_mongo_model
|
17
|
+
MongoModel::EventLog.new DaemonEnvironment.event_store_mongodb
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -1,47 +1,16 @@
|
|
1
1
|
module Euston
|
2
2
|
module MessageBufferDaemon
|
3
|
-
class Daemon
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
:command_buffer_handler => BufferComponent.command_component(AMQP::Channel.new, AMQP::Channel.new),
|
15
|
-
:event_buffer_handler => BufferComponent.event_component(AMQP::Channel.new, AMQP::Channel.new),
|
16
|
-
:event_store_dispatcher => EventStoreComponent.new(AMQP::Channel.new)
|
17
|
-
}
|
18
|
-
end
|
19
|
-
|
20
|
-
def run
|
21
|
-
Thread.abort_on_exception = true
|
22
|
-
|
23
|
-
@clients.each do |name, component|
|
24
|
-
EUSTON_LOG.debug("Starting component: #{name}")
|
25
|
-
component.daemon = self
|
26
|
-
component.start
|
27
|
-
sleep 0.350
|
28
|
-
end
|
29
|
-
|
30
|
-
@clients.each do |name, component|
|
31
|
-
EUSTON_LOG.debug("Thread state of #{name}: #{component.thread_state}")
|
32
|
-
end
|
33
|
-
|
34
|
-
EUSTON_LOG.debug "Components started"
|
35
|
-
|
36
|
-
@queue.pop #<-------- stops here until interrupted
|
37
|
-
|
38
|
-
@clients.each do |name, component|
|
39
|
-
EUSTON_LOG.debug "Stopping component: #{name}"
|
40
|
-
component.stop
|
41
|
-
end
|
42
|
-
|
43
|
-
report_shutdown_reasons
|
44
|
-
EUSTON_LOG.debug "Components stopped"
|
3
|
+
class Daemon < Euston::Daemon
|
4
|
+
private
|
5
|
+
|
6
|
+
def register_components
|
7
|
+
register_component :commands_logger, CommandLogger.new(AMQP::Channel.new)
|
8
|
+
register_component :events_logger, EventLogger.new(AMQP::Channel.new)
|
9
|
+
register_component :command_buffer_publisher, CommandBufferPublisher.new(AMQP::Channel.new)
|
10
|
+
register_component :command_buffer_cleanup, CommandBufferCleanup.new(AMQP::Channel.new)
|
11
|
+
register_component :event_buffer_publisher, EventBufferPublisher.new(AMQP::Channel.new)
|
12
|
+
register_component :event_buffer_cleanup, EventBufferCleanup.new(AMQP::Channel.new)
|
13
|
+
register_component :event_store_dispatcher, EventStoreDispatcher.new
|
45
14
|
end
|
46
15
|
end
|
47
16
|
end
|