euston-daemons 1.0.4-java → 1.1.0-java
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile +3 -1
- data/Rakefile +2 -3
- data/euston-daemons.gemspec +61 -26
- data/lib/euston-daemons/command_processor_daemon/config/environment.rb +27 -18
- data/lib/euston-daemons/command_processor_daemon/lib/clients/command_handler.rb +37 -0
- data/lib/euston-daemons/command_processor_daemon/lib/command_handlers/retry_failed_message.rb +35 -0
- data/lib/euston-daemons/command_processor_daemon/lib/daemon.rb +11 -32
- data/lib/euston-daemons/command_processor_daemon/lib/mongo_models/failed_message.rb +34 -0
- data/lib/euston-daemons/command_processor_daemon/rake_task.rb +15 -13
- data/lib/euston-daemons/euston/daemon.rb +93 -0
- data/lib/euston-daemons/euston/daemon_client.rb +24 -0
- data/lib/euston-daemons/euston/daemon_component.rb +65 -0
- data/lib/euston-daemons/euston/daemon_environment.rb +54 -0
- data/lib/euston-daemons/event_processor_daemon/config/environment.rb +27 -16
- data/lib/euston-daemons/event_processor_daemon/lib/clients/event_handler.rb +42 -0
- data/lib/euston-daemons/event_processor_daemon/lib/daemon.rb +13 -60
- data/lib/euston-daemons/event_processor_daemon/lib/event_handlers/message_failure.rb +27 -0
- data/lib/euston-daemons/event_processor_daemon/rake_task.rb +18 -16
- data/lib/euston-daemons/message_buffer_daemon/config/environment.rb +39 -25
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/command_buffer_cleanup.rb +9 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/command_buffer_publisher.rb +9 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/command_logger.rb +9 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/euston_exchange_accessors.rb +15 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/event_buffer_cleanup.rb +9 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/event_buffer_publisher.rb +9 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/event_logger.rb +9 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/event_store_dispatcher.rb +25 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/message_buffer_cleanup.rb +52 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/message_buffer_publisher.rb +37 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/message_logger.rb +45 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/mongo_model_accessors.rb +21 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/daemon.rb +11 -42
- data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/command_buffer.rb +11 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/command_log.rb +11 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/event_buffer.rb +11 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/event_log.rb +11 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/message_buffer.rb +57 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/{read_model → mongo_models}/message_log.rb +4 -11
- data/lib/euston-daemons/message_buffer_daemon/rake_task.rb +13 -11
- data/lib/euston-daemons/rake_task.rb +41 -65
- data/lib/euston-daemons/rake_tasks.rb +5 -5
- data/lib/euston-daemons/snapshot_daemon/lib/clients/snapshotter.rb +43 -0
- data/lib/euston-daemons/version.rb +1 -1
- data/lib/euston-daemons.rb +6 -1
- data/sample/Rakefile +63 -0
- data/sample/amqp_config.yml +14 -0
- data/sample/command_handlers.rb +17 -0
- data/sample/command_processor_daemon_config.yml +9 -0
- data/sample/event_handlers.rb +21 -0
- data/sample/event_processor_daemon_config.yml +8 -0
- data/sample/message_buffer_daemon_config.yml +8 -0
- data/sample/mongoid_config.yml +13 -0
- data/sample/pids/.placeholder +0 -0
- data/spec/daemons/command_buffer_publisher_spec.rb +110 -0
- data/spec/daemons/command_handler_spec.rb +48 -0
- data/spec/daemons/event_handler_spec.rb +55 -0
- data/spec/daemons/snapshot_client_spec.rb +98 -0
- data/spec/spec_helper.rb +77 -0
- data/spec/support/factories/commands.rb +16 -0
- data/spec/support/factories/commit.rb +7 -0
- data/spec/support/factories/event_message.rb +12 -0
- data/spec/support/factories/events.rb +8 -0
- data/spec/support/filters.rb +14 -0
- data/spec/support/sample_model/commands.rb +14 -0
- data/spec/support/sample_model/counter.rb +36 -0
- data/spec/support/sample_model/counter2.rb +46 -0
- data/spec/support/stub_retrying_subscription.rb +9 -0
- metadata +134 -67
- data/lib/euston-daemons/command_processor_daemon/lib/components/command_handler_component.rb +0 -56
- data/lib/euston-daemons/command_processor_daemon/lib/settings.rb +0 -22
- data/lib/euston-daemons/event_processor_daemon/lib/components/event_handler_component.rb +0 -58
- data/lib/euston-daemons/event_processor_daemon/lib/settings.rb +0 -26
- data/lib/euston-daemons/framework/basic_component.rb +0 -33
- data/lib/euston-daemons/framework/channel_thread.rb +0 -22
- data/lib/euston-daemons/framework/component_shutdown.rb +0 -22
- data/lib/euston-daemons/framework/daemon.rb +0 -27
- data/lib/euston-daemons/framework/handler_bindings_component.rb +0 -56
- data/lib/euston-daemons/framework/queue.rb +0 -71
- data/lib/euston-daemons/message_buffer_daemon/lib/components/buffer_component.rb +0 -73
- data/lib/euston-daemons/message_buffer_daemon/lib/components/event_store_component.rb +0 -52
- data/lib/euston-daemons/message_buffer_daemon/lib/message_logger.rb +0 -54
- data/lib/euston-daemons/message_buffer_daemon/lib/publisher.rb +0 -56
- data/lib/euston-daemons/message_buffer_daemon/lib/settings.rb +0 -14
- data/lib/euston-daemons/message_buffer_daemon/lib/subscriber.rb +0 -60
@@ -1,71 +0,0 @@
|
|
1
|
-
class RabbitMQClient
|
2
|
-
class Queue
|
3
|
-
include Hollywood
|
4
|
-
|
5
|
-
attr_writer :timeout
|
6
|
-
|
7
|
-
def delivery_timeout
|
8
|
-
@timeout ||= 500
|
9
|
-
end
|
10
|
-
|
11
|
-
def safe_subscribe
|
12
|
-
_consumer = self.consumer
|
13
|
-
until Thread.current[:stop] do
|
14
|
-
safe_subscribe_with_timeout(_consumer, self.delivery_timeout)
|
15
|
-
end
|
16
|
-
end
|
17
|
-
|
18
|
-
def safe_handle_message(msg)
|
19
|
-
begin
|
20
|
-
message = JSON.parse_sym(msg.body)
|
21
|
-
begin
|
22
|
-
callback :message_received, message
|
23
|
-
msg.ack!
|
24
|
-
rescue Euston::EventStore::ConcurrencyError
|
25
|
-
msg.reject! true #requeue
|
26
|
-
rescue => e
|
27
|
-
callback :message_failed, message, e, msg
|
28
|
-
Safely.report! e
|
29
|
-
end
|
30
|
-
rescue => e
|
31
|
-
callback :message_decode_failed, msg.body, e
|
32
|
-
msg.ack!
|
33
|
-
Safely.report! e
|
34
|
-
end
|
35
|
-
end
|
36
|
-
|
37
|
-
def consumer(auto_ack=false)
|
38
|
-
consumer = QueueingConsumer.new(@channel)
|
39
|
-
@channel.basic_consume(@name, auto_ack, consumer)
|
40
|
-
consumer
|
41
|
-
end
|
42
|
-
|
43
|
-
def safe_subscribe_with_timeout(consumer, timeout=500)
|
44
|
-
loop do
|
45
|
-
delivery = nil
|
46
|
-
begin
|
47
|
-
delivery = consumer.next_delivery(timeout)
|
48
|
-
rescue NativeException => e
|
49
|
-
Thread.current[:exception] = e
|
50
|
-
break
|
51
|
-
end
|
52
|
-
break if delivery.nil?
|
53
|
-
remsg = ReactiveMessage.new(@channel, delivery, String.from_java_bytes(delivery.get_body))
|
54
|
-
safe_handle_message remsg
|
55
|
-
@channel.basic_ack(delivery.envelope.delivery_tag, false) if remsg.should_acknowledge?
|
56
|
-
end
|
57
|
-
end
|
58
|
-
end
|
59
|
-
|
60
|
-
class ReactiveMessage
|
61
|
-
def reject(opts={})
|
62
|
-
self.reject!(opts.fetch(:requeue, true))
|
63
|
-
end
|
64
|
-
def ack(multiple=false)
|
65
|
-
self.ack!
|
66
|
-
end
|
67
|
-
def method()
|
68
|
-
self.envelope
|
69
|
-
end
|
70
|
-
end
|
71
|
-
end
|
@@ -1,73 +0,0 @@
|
|
1
|
-
module Euston
|
2
|
-
module MessageBufferDaemon
|
3
|
-
class BufferComponent
|
4
|
-
include Euston::Daemons::ComponentShutdown
|
5
|
-
|
6
|
-
class << self
|
7
|
-
def command_component pub_channel, sub_channel
|
8
|
-
self.new Publisher.commands_buffer(pub_channel), Subscriber.commands_buffer(sub_channel)
|
9
|
-
end
|
10
|
-
|
11
|
-
def event_component pub_channel, sub_channel
|
12
|
-
self.new Publisher.events_buffer(pub_channel), Subscriber.events_buffer(sub_channel)
|
13
|
-
end
|
14
|
-
end
|
15
|
-
|
16
|
-
attr_reader :pub_thread, :sub_thread
|
17
|
-
attr_writer :wait_time, :subscribe_timeout
|
18
|
-
|
19
|
-
def initialize pub_buffer, sub_buffer
|
20
|
-
@pub_buffer = pub_buffer
|
21
|
-
@sub_buffer = sub_buffer
|
22
|
-
end
|
23
|
-
|
24
|
-
def wait_time
|
25
|
-
@wait_time ||= 0.2
|
26
|
-
end
|
27
|
-
|
28
|
-
def subscribe_timeout
|
29
|
-
@subscribe_timeout ||= 2000
|
30
|
-
end
|
31
|
-
|
32
|
-
def thread_state
|
33
|
-
{:publish => @pub_thread.status, :subscribe => @sub_thread.status}
|
34
|
-
end
|
35
|
-
|
36
|
-
def stop
|
37
|
-
@pub_thread[:stop], @sub_thread[:stop] = true, true
|
38
|
-
@pub_buffer.disconnect
|
39
|
-
@sub_buffer.disconnect
|
40
|
-
end
|
41
|
-
|
42
|
-
def start
|
43
|
-
@pub_thread = Thread.new do
|
44
|
-
until Thread.current[:stop] do
|
45
|
-
begin
|
46
|
-
@msg_dispatched_count = 0
|
47
|
-
begin
|
48
|
-
@msg_dispatched_count = @pub_buffer.dispatch_due_messages
|
49
|
-
check_exception_and_shutdown
|
50
|
-
end until Thread.current[:stop] || @msg_dispatched_count.zero?
|
51
|
-
rescue => e
|
52
|
-
Thread.current[:exception] = e
|
53
|
-
end
|
54
|
-
check_exception_and_shutdown
|
55
|
-
sleep self.wait_time
|
56
|
-
end
|
57
|
-
end
|
58
|
-
|
59
|
-
@sub_thread = Thread.new do
|
60
|
-
until Thread.current[:stop] do
|
61
|
-
begin
|
62
|
-
@sub_buffer.dequeue_buffered_messages(self.subscribe_timeout)
|
63
|
-
check_exception_and_shutdown
|
64
|
-
rescue => e
|
65
|
-
Thread.current[:exception] = e
|
66
|
-
end
|
67
|
-
check_exception_and_shutdown
|
68
|
-
end
|
69
|
-
end
|
70
|
-
end
|
71
|
-
end
|
72
|
-
end
|
73
|
-
end
|
@@ -1,52 +0,0 @@
|
|
1
|
-
module Euston
|
2
|
-
module MessageBufferDaemon
|
3
|
-
class EventStoreComponent
|
4
|
-
include Euston::Daemons::ComponentShutdown
|
5
|
-
|
6
|
-
attr_reader :cli_thread
|
7
|
-
attr_writer :wait_time
|
8
|
-
|
9
|
-
def initialize channel
|
10
|
-
@channel = channel
|
11
|
-
end
|
12
|
-
|
13
|
-
def wait_time
|
14
|
-
@wait_time ||= 0.2
|
15
|
-
end
|
16
|
-
|
17
|
-
def start
|
18
|
-
@cli_thread = Thread.new do
|
19
|
-
@event_buffer = Euston::RabbitMq::ReadModel::MessageBuffer.events
|
20
|
-
@event_store = Euston::RabbitMq.event_store
|
21
|
-
|
22
|
-
until Thread.current[:stop] do
|
23
|
-
begin
|
24
|
-
loop do
|
25
|
-
commits = @event_store.get_undispatched_commits
|
26
|
-
break if commits.empty?
|
27
|
-
commits.each do |commit|
|
28
|
-
commit.events.each { |event| @event_buffer.buffer_new_message event.to_hash }
|
29
|
-
@event_store.mark_commit_as_dispatched commit
|
30
|
-
end
|
31
|
-
end
|
32
|
-
rescue => e
|
33
|
-
Thread.current[:exception] = e
|
34
|
-
end
|
35
|
-
check_exception_and_shutdown
|
36
|
-
sleep self.wait_time
|
37
|
-
end
|
38
|
-
end
|
39
|
-
end
|
40
|
-
|
41
|
-
def thread_state
|
42
|
-
@cli_thread.status
|
43
|
-
end
|
44
|
-
|
45
|
-
def stop
|
46
|
-
@cli_thread[:stop] = true
|
47
|
-
@channel.disconnect
|
48
|
-
end
|
49
|
-
|
50
|
-
end
|
51
|
-
end
|
52
|
-
end
|
@@ -1,54 +0,0 @@
|
|
1
|
-
module Euston
|
2
|
-
class MessageLogger
|
3
|
-
class << self
|
4
|
-
def commands_logger channel
|
5
|
-
self.new channel, :commands
|
6
|
-
end
|
7
|
-
|
8
|
-
def events_logger channel
|
9
|
-
self.new channel, :events
|
10
|
-
end
|
11
|
-
end
|
12
|
-
|
13
|
-
include Euston::RabbitMq::Exchanges
|
14
|
-
include Euston::RabbitMq::Queues
|
15
|
-
|
16
|
-
def initialize channel, exchange_name
|
17
|
-
@channel = channel
|
18
|
-
@exchange = get_exchange channel, exchange_name
|
19
|
-
@read_model = Euston::MessageBufferDaemon::ReadModel::MessageLog.send exchange_name
|
20
|
-
|
21
|
-
@queue = get_queue channel, "#{exchange_name}_log"
|
22
|
-
@queue.bind @exchange, :routing_key => "#{exchange_name}.#"
|
23
|
-
|
24
|
-
@queue.when(:message_decode_failed => method(:log_failure),
|
25
|
-
:message_failed => method(:message_failed),
|
26
|
-
:message_received => method(:write_message_to_log))
|
27
|
-
end
|
28
|
-
|
29
|
-
def message_failed(message, error, header)
|
30
|
-
log_failure message, error
|
31
|
-
header.ack
|
32
|
-
end
|
33
|
-
|
34
|
-
def disconnect
|
35
|
-
@channel.disconnect
|
36
|
-
end
|
37
|
-
|
38
|
-
def start
|
39
|
-
@queue.safe_subscribe
|
40
|
-
end
|
41
|
-
|
42
|
-
def log_failure message, error
|
43
|
-
text = "A log queue subscription failed. [Error] #{error.message} [Payload] #{message}"
|
44
|
-
err = Euston::RabbitMq::MessageDecodeFailedError.new text
|
45
|
-
err.set_backtrace error.backtrace
|
46
|
-
|
47
|
-
Safely.report! err
|
48
|
-
end
|
49
|
-
|
50
|
-
def write_message_to_log message
|
51
|
-
@read_model.log_new_message message
|
52
|
-
end
|
53
|
-
end
|
54
|
-
end
|
@@ -1,56 +0,0 @@
|
|
1
|
-
module Euston
|
2
|
-
module MessageBufferDaemon
|
3
|
-
class Publisher
|
4
|
-
class << self
|
5
|
-
def commands_buffer channel
|
6
|
-
self.new channel, :commands
|
7
|
-
end
|
8
|
-
|
9
|
-
def events_buffer channel
|
10
|
-
self.new channel, :events
|
11
|
-
end
|
12
|
-
end
|
13
|
-
|
14
|
-
include Euston::RabbitMq::Exchanges
|
15
|
-
|
16
|
-
def initialize channel, exchange_name
|
17
|
-
@read_model = Euston::RabbitMq::ReadModel::MessageBuffer.send exchange_name
|
18
|
-
|
19
|
-
@channel = channel
|
20
|
-
@exchange = get_exchange @channel, exchange_name
|
21
|
-
end
|
22
|
-
|
23
|
-
def disconnect
|
24
|
-
@channel.disconnect
|
25
|
-
end
|
26
|
-
|
27
|
-
def dispatch_one_due_message
|
28
|
-
if (message = @read_model.find_next_message)
|
29
|
-
@read_model.set_next_attempt message
|
30
|
-
protected_publish(message)
|
31
|
-
end
|
32
|
-
end
|
33
|
-
|
34
|
-
def dispatch_due_messages
|
35
|
-
dispatched = 0
|
36
|
-
@read_model.find_due_messages.each do |message|
|
37
|
-
@read_model.set_next_attempt message
|
38
|
-
break unless protected_publish(message)
|
39
|
-
dispatched += 1
|
40
|
-
end
|
41
|
-
dispatched
|
42
|
-
end
|
43
|
-
|
44
|
-
def protected_publish(message)
|
45
|
-
ret = true
|
46
|
-
begin
|
47
|
-
@exchange.publish message['json'], default_publish_options.merge(:routing_key => "#{@exchange.name}.#{message['type']}")
|
48
|
-
rescue NativeException => e
|
49
|
-
Thread.current[:exception] = e
|
50
|
-
ret = false
|
51
|
-
end
|
52
|
-
ret
|
53
|
-
end
|
54
|
-
end
|
55
|
-
end
|
56
|
-
end
|
@@ -1,60 +0,0 @@
|
|
1
|
-
module Euston
|
2
|
-
module MessageBufferDaemon
|
3
|
-
class Subscriber
|
4
|
-
class << self
|
5
|
-
def commands_buffer channel
|
6
|
-
self.new channel, :commands
|
7
|
-
end
|
8
|
-
|
9
|
-
def events_buffer channel
|
10
|
-
self.new channel, :events
|
11
|
-
end
|
12
|
-
end
|
13
|
-
|
14
|
-
include Euston::RabbitMq::Exchanges
|
15
|
-
include Euston::RabbitMq::Queues
|
16
|
-
include Hollywood
|
17
|
-
|
18
|
-
def initialize channel, exchange_name
|
19
|
-
@read_model = Euston::RabbitMq::ReadModel::MessageBuffer.send exchange_name
|
20
|
-
|
21
|
-
@channel = channel
|
22
|
-
@channel.prefetch(1)
|
23
|
-
@exchange = get_exchange @channel, exchange_name
|
24
|
-
@queue = get_queue @channel, "#{exchange_name}_buffer"
|
25
|
-
@queue.bind @exchange, :routing_key => "#{exchange_name}.#"
|
26
|
-
|
27
|
-
@queue.when(:message_decode_failed => method(:log_failure),
|
28
|
-
:message_failed => method(:message_failed),
|
29
|
-
:message_received => method(:remove_message_from_buffer))
|
30
|
-
|
31
|
-
@consumer = @queue.consumer
|
32
|
-
end
|
33
|
-
|
34
|
-
def disconnect
|
35
|
-
@channel.disconnect
|
36
|
-
end
|
37
|
-
|
38
|
-
def dequeue_buffered_messages(timeout) #will block until message or timeout
|
39
|
-
@queue.safe_subscribe_with_timeout(@consumer, timeout)
|
40
|
-
end
|
41
|
-
|
42
|
-
def log_failure message, error
|
43
|
-
text = "A buffer queue subscription failed. [Error] #{error.message} [Payload] #{message}"
|
44
|
-
err = Euston::RabbitMq::MessageDecodeFailedError.new text
|
45
|
-
err.set_backtrace error.backtrace
|
46
|
-
|
47
|
-
Safely.report! err
|
48
|
-
end
|
49
|
-
|
50
|
-
def message_failed(message, error, header)
|
51
|
-
header.ack!
|
52
|
-
log_failure message, error
|
53
|
-
end
|
54
|
-
|
55
|
-
def remove_message_from_buffer message
|
56
|
-
@read_model.remove_published_message message[:headers][:id]
|
57
|
-
end
|
58
|
-
end
|
59
|
-
end
|
60
|
-
end
|