euston-daemons 1.0.4-java → 1.1.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +3 -1
- data/Rakefile +2 -3
- data/euston-daemons.gemspec +61 -26
- data/lib/euston-daemons/command_processor_daemon/config/environment.rb +27 -18
- data/lib/euston-daemons/command_processor_daemon/lib/clients/command_handler.rb +37 -0
- data/lib/euston-daemons/command_processor_daemon/lib/command_handlers/retry_failed_message.rb +35 -0
- data/lib/euston-daemons/command_processor_daemon/lib/daemon.rb +11 -32
- data/lib/euston-daemons/command_processor_daemon/lib/mongo_models/failed_message.rb +34 -0
- data/lib/euston-daemons/command_processor_daemon/rake_task.rb +15 -13
- data/lib/euston-daemons/euston/daemon.rb +93 -0
- data/lib/euston-daemons/euston/daemon_client.rb +24 -0
- data/lib/euston-daemons/euston/daemon_component.rb +65 -0
- data/lib/euston-daemons/euston/daemon_environment.rb +54 -0
- data/lib/euston-daemons/event_processor_daemon/config/environment.rb +27 -16
- data/lib/euston-daemons/event_processor_daemon/lib/clients/event_handler.rb +42 -0
- data/lib/euston-daemons/event_processor_daemon/lib/daemon.rb +13 -60
- data/lib/euston-daemons/event_processor_daemon/lib/event_handlers/message_failure.rb +27 -0
- data/lib/euston-daemons/event_processor_daemon/rake_task.rb +18 -16
- data/lib/euston-daemons/message_buffer_daemon/config/environment.rb +39 -25
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/command_buffer_cleanup.rb +9 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/command_buffer_publisher.rb +9 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/command_logger.rb +9 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/euston_exchange_accessors.rb +15 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/event_buffer_cleanup.rb +9 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/event_buffer_publisher.rb +9 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/event_logger.rb +9 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/event_store_dispatcher.rb +25 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/message_buffer_cleanup.rb +52 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/message_buffer_publisher.rb +37 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/message_logger.rb +45 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/clients/mongo_model_accessors.rb +21 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/daemon.rb +11 -42
- data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/command_buffer.rb +11 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/command_log.rb +11 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/event_buffer.rb +11 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/event_log.rb +11 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/message_buffer.rb +57 -0
- data/lib/euston-daemons/message_buffer_daemon/lib/{read_model → mongo_models}/message_log.rb +4 -11
- data/lib/euston-daemons/message_buffer_daemon/rake_task.rb +13 -11
- data/lib/euston-daemons/rake_task.rb +41 -65
- data/lib/euston-daemons/rake_tasks.rb +5 -5
- data/lib/euston-daemons/snapshot_daemon/lib/clients/snapshotter.rb +43 -0
- data/lib/euston-daemons/version.rb +1 -1
- data/lib/euston-daemons.rb +6 -1
- data/sample/Rakefile +63 -0
- data/sample/amqp_config.yml +14 -0
- data/sample/command_handlers.rb +17 -0
- data/sample/command_processor_daemon_config.yml +9 -0
- data/sample/event_handlers.rb +21 -0
- data/sample/event_processor_daemon_config.yml +8 -0
- data/sample/message_buffer_daemon_config.yml +8 -0
- data/sample/mongoid_config.yml +13 -0
- data/sample/pids/.placeholder +0 -0
- data/spec/daemons/command_buffer_publisher_spec.rb +110 -0
- data/spec/daemons/command_handler_spec.rb +48 -0
- data/spec/daemons/event_handler_spec.rb +55 -0
- data/spec/daemons/snapshot_client_spec.rb +98 -0
- data/spec/spec_helper.rb +77 -0
- data/spec/support/factories/commands.rb +16 -0
- data/spec/support/factories/commit.rb +7 -0
- data/spec/support/factories/event_message.rb +12 -0
- data/spec/support/factories/events.rb +8 -0
- data/spec/support/filters.rb +14 -0
- data/spec/support/sample_model/commands.rb +14 -0
- data/spec/support/sample_model/counter.rb +36 -0
- data/spec/support/sample_model/counter2.rb +46 -0
- data/spec/support/stub_retrying_subscription.rb +9 -0
- metadata +134 -67
- data/lib/euston-daemons/command_processor_daemon/lib/components/command_handler_component.rb +0 -56
- data/lib/euston-daemons/command_processor_daemon/lib/settings.rb +0 -22
- data/lib/euston-daemons/event_processor_daemon/lib/components/event_handler_component.rb +0 -58
- data/lib/euston-daemons/event_processor_daemon/lib/settings.rb +0 -26
- data/lib/euston-daemons/framework/basic_component.rb +0 -33
- data/lib/euston-daemons/framework/channel_thread.rb +0 -22
- data/lib/euston-daemons/framework/component_shutdown.rb +0 -22
- data/lib/euston-daemons/framework/daemon.rb +0 -27
- data/lib/euston-daemons/framework/handler_bindings_component.rb +0 -56
- data/lib/euston-daemons/framework/queue.rb +0 -71
- data/lib/euston-daemons/message_buffer_daemon/lib/components/buffer_component.rb +0 -73
- data/lib/euston-daemons/message_buffer_daemon/lib/components/event_store_component.rb +0 -52
- data/lib/euston-daemons/message_buffer_daemon/lib/message_logger.rb +0 -54
- data/lib/euston-daemons/message_buffer_daemon/lib/publisher.rb +0 -56
- data/lib/euston-daemons/message_buffer_daemon/lib/settings.rb +0 -14
- data/lib/euston-daemons/message_buffer_daemon/lib/subscriber.rb +0 -60
|
@@ -1,71 +0,0 @@
|
|
|
1
|
-
class RabbitMQClient
|
|
2
|
-
class Queue
|
|
3
|
-
include Hollywood
|
|
4
|
-
|
|
5
|
-
attr_writer :timeout
|
|
6
|
-
|
|
7
|
-
def delivery_timeout
|
|
8
|
-
@timeout ||= 500
|
|
9
|
-
end
|
|
10
|
-
|
|
11
|
-
def safe_subscribe
|
|
12
|
-
_consumer = self.consumer
|
|
13
|
-
until Thread.current[:stop] do
|
|
14
|
-
safe_subscribe_with_timeout(_consumer, self.delivery_timeout)
|
|
15
|
-
end
|
|
16
|
-
end
|
|
17
|
-
|
|
18
|
-
def safe_handle_message(msg)
|
|
19
|
-
begin
|
|
20
|
-
message = JSON.parse_sym(msg.body)
|
|
21
|
-
begin
|
|
22
|
-
callback :message_received, message
|
|
23
|
-
msg.ack!
|
|
24
|
-
rescue Euston::EventStore::ConcurrencyError
|
|
25
|
-
msg.reject! true #requeue
|
|
26
|
-
rescue => e
|
|
27
|
-
callback :message_failed, message, e, msg
|
|
28
|
-
Safely.report! e
|
|
29
|
-
end
|
|
30
|
-
rescue => e
|
|
31
|
-
callback :message_decode_failed, msg.body, e
|
|
32
|
-
msg.ack!
|
|
33
|
-
Safely.report! e
|
|
34
|
-
end
|
|
35
|
-
end
|
|
36
|
-
|
|
37
|
-
def consumer(auto_ack=false)
|
|
38
|
-
consumer = QueueingConsumer.new(@channel)
|
|
39
|
-
@channel.basic_consume(@name, auto_ack, consumer)
|
|
40
|
-
consumer
|
|
41
|
-
end
|
|
42
|
-
|
|
43
|
-
def safe_subscribe_with_timeout(consumer, timeout=500)
|
|
44
|
-
loop do
|
|
45
|
-
delivery = nil
|
|
46
|
-
begin
|
|
47
|
-
delivery = consumer.next_delivery(timeout)
|
|
48
|
-
rescue NativeException => e
|
|
49
|
-
Thread.current[:exception] = e
|
|
50
|
-
break
|
|
51
|
-
end
|
|
52
|
-
break if delivery.nil?
|
|
53
|
-
remsg = ReactiveMessage.new(@channel, delivery, String.from_java_bytes(delivery.get_body))
|
|
54
|
-
safe_handle_message remsg
|
|
55
|
-
@channel.basic_ack(delivery.envelope.delivery_tag, false) if remsg.should_acknowledge?
|
|
56
|
-
end
|
|
57
|
-
end
|
|
58
|
-
end
|
|
59
|
-
|
|
60
|
-
class ReactiveMessage
|
|
61
|
-
def reject(opts={})
|
|
62
|
-
self.reject!(opts.fetch(:requeue, true))
|
|
63
|
-
end
|
|
64
|
-
def ack(multiple=false)
|
|
65
|
-
self.ack!
|
|
66
|
-
end
|
|
67
|
-
def method()
|
|
68
|
-
self.envelope
|
|
69
|
-
end
|
|
70
|
-
end
|
|
71
|
-
end
|
|
@@ -1,73 +0,0 @@
|
|
|
1
|
-
module Euston
|
|
2
|
-
module MessageBufferDaemon
|
|
3
|
-
class BufferComponent
|
|
4
|
-
include Euston::Daemons::ComponentShutdown
|
|
5
|
-
|
|
6
|
-
class << self
|
|
7
|
-
def command_component pub_channel, sub_channel
|
|
8
|
-
self.new Publisher.commands_buffer(pub_channel), Subscriber.commands_buffer(sub_channel)
|
|
9
|
-
end
|
|
10
|
-
|
|
11
|
-
def event_component pub_channel, sub_channel
|
|
12
|
-
self.new Publisher.events_buffer(pub_channel), Subscriber.events_buffer(sub_channel)
|
|
13
|
-
end
|
|
14
|
-
end
|
|
15
|
-
|
|
16
|
-
attr_reader :pub_thread, :sub_thread
|
|
17
|
-
attr_writer :wait_time, :subscribe_timeout
|
|
18
|
-
|
|
19
|
-
def initialize pub_buffer, sub_buffer
|
|
20
|
-
@pub_buffer = pub_buffer
|
|
21
|
-
@sub_buffer = sub_buffer
|
|
22
|
-
end
|
|
23
|
-
|
|
24
|
-
def wait_time
|
|
25
|
-
@wait_time ||= 0.2
|
|
26
|
-
end
|
|
27
|
-
|
|
28
|
-
def subscribe_timeout
|
|
29
|
-
@subscribe_timeout ||= 2000
|
|
30
|
-
end
|
|
31
|
-
|
|
32
|
-
def thread_state
|
|
33
|
-
{:publish => @pub_thread.status, :subscribe => @sub_thread.status}
|
|
34
|
-
end
|
|
35
|
-
|
|
36
|
-
def stop
|
|
37
|
-
@pub_thread[:stop], @sub_thread[:stop] = true, true
|
|
38
|
-
@pub_buffer.disconnect
|
|
39
|
-
@sub_buffer.disconnect
|
|
40
|
-
end
|
|
41
|
-
|
|
42
|
-
def start
|
|
43
|
-
@pub_thread = Thread.new do
|
|
44
|
-
until Thread.current[:stop] do
|
|
45
|
-
begin
|
|
46
|
-
@msg_dispatched_count = 0
|
|
47
|
-
begin
|
|
48
|
-
@msg_dispatched_count = @pub_buffer.dispatch_due_messages
|
|
49
|
-
check_exception_and_shutdown
|
|
50
|
-
end until Thread.current[:stop] || @msg_dispatched_count.zero?
|
|
51
|
-
rescue => e
|
|
52
|
-
Thread.current[:exception] = e
|
|
53
|
-
end
|
|
54
|
-
check_exception_and_shutdown
|
|
55
|
-
sleep self.wait_time
|
|
56
|
-
end
|
|
57
|
-
end
|
|
58
|
-
|
|
59
|
-
@sub_thread = Thread.new do
|
|
60
|
-
until Thread.current[:stop] do
|
|
61
|
-
begin
|
|
62
|
-
@sub_buffer.dequeue_buffered_messages(self.subscribe_timeout)
|
|
63
|
-
check_exception_and_shutdown
|
|
64
|
-
rescue => e
|
|
65
|
-
Thread.current[:exception] = e
|
|
66
|
-
end
|
|
67
|
-
check_exception_and_shutdown
|
|
68
|
-
end
|
|
69
|
-
end
|
|
70
|
-
end
|
|
71
|
-
end
|
|
72
|
-
end
|
|
73
|
-
end
|
|
@@ -1,52 +0,0 @@
|
|
|
1
|
-
module Euston
|
|
2
|
-
module MessageBufferDaemon
|
|
3
|
-
class EventStoreComponent
|
|
4
|
-
include Euston::Daemons::ComponentShutdown
|
|
5
|
-
|
|
6
|
-
attr_reader :cli_thread
|
|
7
|
-
attr_writer :wait_time
|
|
8
|
-
|
|
9
|
-
def initialize channel
|
|
10
|
-
@channel = channel
|
|
11
|
-
end
|
|
12
|
-
|
|
13
|
-
def wait_time
|
|
14
|
-
@wait_time ||= 0.2
|
|
15
|
-
end
|
|
16
|
-
|
|
17
|
-
def start
|
|
18
|
-
@cli_thread = Thread.new do
|
|
19
|
-
@event_buffer = Euston::RabbitMq::ReadModel::MessageBuffer.events
|
|
20
|
-
@event_store = Euston::RabbitMq.event_store
|
|
21
|
-
|
|
22
|
-
until Thread.current[:stop] do
|
|
23
|
-
begin
|
|
24
|
-
loop do
|
|
25
|
-
commits = @event_store.get_undispatched_commits
|
|
26
|
-
break if commits.empty?
|
|
27
|
-
commits.each do |commit|
|
|
28
|
-
commit.events.each { |event| @event_buffer.buffer_new_message event.to_hash }
|
|
29
|
-
@event_store.mark_commit_as_dispatched commit
|
|
30
|
-
end
|
|
31
|
-
end
|
|
32
|
-
rescue => e
|
|
33
|
-
Thread.current[:exception] = e
|
|
34
|
-
end
|
|
35
|
-
check_exception_and_shutdown
|
|
36
|
-
sleep self.wait_time
|
|
37
|
-
end
|
|
38
|
-
end
|
|
39
|
-
end
|
|
40
|
-
|
|
41
|
-
def thread_state
|
|
42
|
-
@cli_thread.status
|
|
43
|
-
end
|
|
44
|
-
|
|
45
|
-
def stop
|
|
46
|
-
@cli_thread[:stop] = true
|
|
47
|
-
@channel.disconnect
|
|
48
|
-
end
|
|
49
|
-
|
|
50
|
-
end
|
|
51
|
-
end
|
|
52
|
-
end
|
|
@@ -1,54 +0,0 @@
|
|
|
1
|
-
module Euston
|
|
2
|
-
class MessageLogger
|
|
3
|
-
class << self
|
|
4
|
-
def commands_logger channel
|
|
5
|
-
self.new channel, :commands
|
|
6
|
-
end
|
|
7
|
-
|
|
8
|
-
def events_logger channel
|
|
9
|
-
self.new channel, :events
|
|
10
|
-
end
|
|
11
|
-
end
|
|
12
|
-
|
|
13
|
-
include Euston::RabbitMq::Exchanges
|
|
14
|
-
include Euston::RabbitMq::Queues
|
|
15
|
-
|
|
16
|
-
def initialize channel, exchange_name
|
|
17
|
-
@channel = channel
|
|
18
|
-
@exchange = get_exchange channel, exchange_name
|
|
19
|
-
@read_model = Euston::MessageBufferDaemon::ReadModel::MessageLog.send exchange_name
|
|
20
|
-
|
|
21
|
-
@queue = get_queue channel, "#{exchange_name}_log"
|
|
22
|
-
@queue.bind @exchange, :routing_key => "#{exchange_name}.#"
|
|
23
|
-
|
|
24
|
-
@queue.when(:message_decode_failed => method(:log_failure),
|
|
25
|
-
:message_failed => method(:message_failed),
|
|
26
|
-
:message_received => method(:write_message_to_log))
|
|
27
|
-
end
|
|
28
|
-
|
|
29
|
-
def message_failed(message, error, header)
|
|
30
|
-
log_failure message, error
|
|
31
|
-
header.ack
|
|
32
|
-
end
|
|
33
|
-
|
|
34
|
-
def disconnect
|
|
35
|
-
@channel.disconnect
|
|
36
|
-
end
|
|
37
|
-
|
|
38
|
-
def start
|
|
39
|
-
@queue.safe_subscribe
|
|
40
|
-
end
|
|
41
|
-
|
|
42
|
-
def log_failure message, error
|
|
43
|
-
text = "A log queue subscription failed. [Error] #{error.message} [Payload] #{message}"
|
|
44
|
-
err = Euston::RabbitMq::MessageDecodeFailedError.new text
|
|
45
|
-
err.set_backtrace error.backtrace
|
|
46
|
-
|
|
47
|
-
Safely.report! err
|
|
48
|
-
end
|
|
49
|
-
|
|
50
|
-
def write_message_to_log message
|
|
51
|
-
@read_model.log_new_message message
|
|
52
|
-
end
|
|
53
|
-
end
|
|
54
|
-
end
|
|
@@ -1,56 +0,0 @@
|
|
|
1
|
-
module Euston
|
|
2
|
-
module MessageBufferDaemon
|
|
3
|
-
class Publisher
|
|
4
|
-
class << self
|
|
5
|
-
def commands_buffer channel
|
|
6
|
-
self.new channel, :commands
|
|
7
|
-
end
|
|
8
|
-
|
|
9
|
-
def events_buffer channel
|
|
10
|
-
self.new channel, :events
|
|
11
|
-
end
|
|
12
|
-
end
|
|
13
|
-
|
|
14
|
-
include Euston::RabbitMq::Exchanges
|
|
15
|
-
|
|
16
|
-
def initialize channel, exchange_name
|
|
17
|
-
@read_model = Euston::RabbitMq::ReadModel::MessageBuffer.send exchange_name
|
|
18
|
-
|
|
19
|
-
@channel = channel
|
|
20
|
-
@exchange = get_exchange @channel, exchange_name
|
|
21
|
-
end
|
|
22
|
-
|
|
23
|
-
def disconnect
|
|
24
|
-
@channel.disconnect
|
|
25
|
-
end
|
|
26
|
-
|
|
27
|
-
def dispatch_one_due_message
|
|
28
|
-
if (message = @read_model.find_next_message)
|
|
29
|
-
@read_model.set_next_attempt message
|
|
30
|
-
protected_publish(message)
|
|
31
|
-
end
|
|
32
|
-
end
|
|
33
|
-
|
|
34
|
-
def dispatch_due_messages
|
|
35
|
-
dispatched = 0
|
|
36
|
-
@read_model.find_due_messages.each do |message|
|
|
37
|
-
@read_model.set_next_attempt message
|
|
38
|
-
break unless protected_publish(message)
|
|
39
|
-
dispatched += 1
|
|
40
|
-
end
|
|
41
|
-
dispatched
|
|
42
|
-
end
|
|
43
|
-
|
|
44
|
-
def protected_publish(message)
|
|
45
|
-
ret = true
|
|
46
|
-
begin
|
|
47
|
-
@exchange.publish message['json'], default_publish_options.merge(:routing_key => "#{@exchange.name}.#{message['type']}")
|
|
48
|
-
rescue NativeException => e
|
|
49
|
-
Thread.current[:exception] = e
|
|
50
|
-
ret = false
|
|
51
|
-
end
|
|
52
|
-
ret
|
|
53
|
-
end
|
|
54
|
-
end
|
|
55
|
-
end
|
|
56
|
-
end
|
|
@@ -1,60 +0,0 @@
|
|
|
1
|
-
module Euston
|
|
2
|
-
module MessageBufferDaemon
|
|
3
|
-
class Subscriber
|
|
4
|
-
class << self
|
|
5
|
-
def commands_buffer channel
|
|
6
|
-
self.new channel, :commands
|
|
7
|
-
end
|
|
8
|
-
|
|
9
|
-
def events_buffer channel
|
|
10
|
-
self.new channel, :events
|
|
11
|
-
end
|
|
12
|
-
end
|
|
13
|
-
|
|
14
|
-
include Euston::RabbitMq::Exchanges
|
|
15
|
-
include Euston::RabbitMq::Queues
|
|
16
|
-
include Hollywood
|
|
17
|
-
|
|
18
|
-
def initialize channel, exchange_name
|
|
19
|
-
@read_model = Euston::RabbitMq::ReadModel::MessageBuffer.send exchange_name
|
|
20
|
-
|
|
21
|
-
@channel = channel
|
|
22
|
-
@channel.prefetch(1)
|
|
23
|
-
@exchange = get_exchange @channel, exchange_name
|
|
24
|
-
@queue = get_queue @channel, "#{exchange_name}_buffer"
|
|
25
|
-
@queue.bind @exchange, :routing_key => "#{exchange_name}.#"
|
|
26
|
-
|
|
27
|
-
@queue.when(:message_decode_failed => method(:log_failure),
|
|
28
|
-
:message_failed => method(:message_failed),
|
|
29
|
-
:message_received => method(:remove_message_from_buffer))
|
|
30
|
-
|
|
31
|
-
@consumer = @queue.consumer
|
|
32
|
-
end
|
|
33
|
-
|
|
34
|
-
def disconnect
|
|
35
|
-
@channel.disconnect
|
|
36
|
-
end
|
|
37
|
-
|
|
38
|
-
def dequeue_buffered_messages(timeout) #will block until message or timeout
|
|
39
|
-
@queue.safe_subscribe_with_timeout(@consumer, timeout)
|
|
40
|
-
end
|
|
41
|
-
|
|
42
|
-
def log_failure message, error
|
|
43
|
-
text = "A buffer queue subscription failed. [Error] #{error.message} [Payload] #{message}"
|
|
44
|
-
err = Euston::RabbitMq::MessageDecodeFailedError.new text
|
|
45
|
-
err.set_backtrace error.backtrace
|
|
46
|
-
|
|
47
|
-
Safely.report! err
|
|
48
|
-
end
|
|
49
|
-
|
|
50
|
-
def message_failed(message, error, header)
|
|
51
|
-
header.ack!
|
|
52
|
-
log_failure message, error
|
|
53
|
-
end
|
|
54
|
-
|
|
55
|
-
def remove_message_from_buffer message
|
|
56
|
-
@read_model.remove_published_message message[:headers][:id]
|
|
57
|
-
end
|
|
58
|
-
end
|
|
59
|
-
end
|
|
60
|
-
end
|