euston-rabbitmq 1.0.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. data/Gemfile +2 -0
  2. data/Gemfile.lock +103 -0
  3. data/Rakefile +182 -0
  4. data/euston-rabbitmq.gemspec +74 -0
  5. data/lib/euston-rabbitmq/buffered_message_dispatcher.rb +48 -0
  6. data/lib/euston-rabbitmq/command_handler_bindings.rb +30 -0
  7. data/lib/euston-rabbitmq/command_handlers/retry_failed_message.rb +29 -0
  8. data/lib/euston-rabbitmq/errors.rb +5 -0
  9. data/lib/euston-rabbitmq/event_handler_binding.rb +114 -0
  10. data/lib/euston-rabbitmq/event_handler_bindings.rb +30 -0
  11. data/lib/euston-rabbitmq/event_handlers/message_failure.rb +27 -0
  12. data/lib/euston-rabbitmq/event_store_dispatcher.rb +45 -0
  13. data/lib/euston-rabbitmq/exchanges.rb +17 -0
  14. data/lib/euston-rabbitmq/handler_bindings.rb +115 -0
  15. data/lib/euston-rabbitmq/message_buffer.rb +68 -0
  16. data/lib/euston-rabbitmq/message_logger.rb +50 -0
  17. data/lib/euston-rabbitmq/queue.rb +30 -0
  18. data/lib/euston-rabbitmq/queues.rb +13 -0
  19. data/lib/euston-rabbitmq/read_model/failed_message.rb +36 -0
  20. data/lib/euston-rabbitmq/read_model/message_buffer.rb +52 -0
  21. data/lib/euston-rabbitmq/read_model/message_log.rb +37 -0
  22. data/lib/euston-rabbitmq/version.rb +5 -0
  23. data/lib/euston-rabbitmq.rb +28 -0
  24. data/spec/command_buffer_spec.rb +69 -0
  25. data/spec/event_buffer_spec.rb +69 -0
  26. data/spec/exchange_declaration_spec.rb +28 -0
  27. data/spec/message_failure_spec.rb +77 -0
  28. data/spec/mt_safe_queue_subscription_spec.rb +72 -0
  29. data/spec/safe_queue_subscription_spec.rb +50 -0
  30. data/spec/spec_helper.rb +65 -0
  31. data/spec/support/factories.rb +18 -0
  32. metadata +233 -0
@@ -0,0 +1,17 @@
1
+ module Euston
2
+ module RabbitMq
3
+ module Exchanges
4
+ def default_exchange_options
5
+ { :durable => true, :nowait => false }
6
+ end
7
+
8
+ def default_publish_options
9
+ { :immediate => false, :mandatory => true, :persistent => true }
10
+ end
11
+
12
+ def get_exchange channel, name, opts = {}
13
+ channel.topic name.to_s, default_exchange_options.merge(opts)
14
+ end
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,115 @@
1
+ module Euston
2
+ module RabbitMq
3
+ class HandlerBindings
4
+ include Euston::RabbitMq::Exchanges
5
+ include Euston::RabbitMq::Queues
6
+
7
+ def initialize channel
8
+ @channel = channel
9
+ @exchanges = { :commands => get_exchange(@channel, :commands),
10
+ :events => get_exchange(@channel, :events) }
11
+ @namespaces = []
12
+ end
13
+
14
+ def add_namespace namespace
15
+ @namespaces << namespace
16
+ end
17
+
18
+ def namespaced_handler_types
19
+ ret = []
20
+ @namespaces.each do |namespace|
21
+ namespace.constants.each do |handler_type|
22
+ ret << [namespace, handler_type]
23
+ end
24
+ end
25
+ ret
26
+ end
27
+
28
+ def finalize_bindings
29
+ namespaced_handler_types.each do |namespace, handler_type|
30
+ bind_handler handler_type
31
+ end
32
+ end
33
+
34
+ protected
35
+
36
+ def bind_handler handler_type
37
+ # virtual
38
+ end
39
+
40
+ def find_namespaces_containing_handler_type handler_type
41
+ @namespaces.select { |namespace| namespace.const_defined? handler_type }
42
+ end
43
+
44
+ def attach_queue_hook_listeners queue
45
+ queue.when(:message_decode_failed => method(:log_decode_failure),
46
+ :message_failed => method(:handle_failure),
47
+ :message_received => method(:call_handler))
48
+
49
+ queue.safe_subscribe
50
+ end
51
+
52
+ def call_handler(message)
53
+ # abstract
54
+ end
55
+
56
+ def create_message_failed_message error, failed_message, amqp_header
57
+ {
58
+ :headers =>
59
+ {
60
+ :id => Euston.uuid.generate.to_s,
61
+ :type => :message_failed,
62
+ :version => 1,
63
+ :timestamp => Time.now.to_f
64
+ },
65
+
66
+ :body =>
67
+ {
68
+ :message => failed_message,
69
+ :routing_key => amqp_header.method.routing_key,
70
+ :error => error.message,
71
+ :backtrace => error.backtrace
72
+ }
73
+ }
74
+ end
75
+
76
+ def handle_failure message, error, amqp_header
77
+ failures = message[:failures] || 0
78
+ failures = failures + 1
79
+ message[:failures] = failures
80
+
81
+ begin
82
+ if failures == 3
83
+ message.delete :failures
84
+ message = create_message_failed_message error, message, amqp_header
85
+
86
+ options = { :key => 'events.message_failed' }
87
+ exchange = :events
88
+ else
89
+ options = { :key => amqp_header.method.routing_key }
90
+ exchange = amqp_header.method.routing_key.split('.').first.to_sym
91
+ end
92
+
93
+ options = default_publish_options.merge options
94
+ exchange = @exchanges[exchange]
95
+ message = ::ActiveSupport::JSON.encode message
96
+
97
+ exchange.publish message, options
98
+
99
+ amqp_header.ack
100
+ rescue StandardError => e
101
+ amqp_header.reject :requeue => true
102
+ raise e
103
+ end
104
+ end
105
+
106
+ def log_decode_failure message, error
107
+ text = "A handler queue subscription failed. [Error] #{error.message} [Payload] #{message}"
108
+ err = Euston::RabbitMq::MessageDecodeFailedError.new text
109
+ err.set_backtrace error.backtrace
110
+
111
+ Safely.report! err
112
+ end
113
+ end
114
+ end
115
+ end
@@ -0,0 +1,68 @@
1
+ module Euston
2
+ module RabbitMq
3
+ class MessageBuffer
4
+ class << self
5
+ def commands_buffer channel = nil
6
+ @command_buffers ||= MessageBuffer.new channel, :commands
7
+ end
8
+
9
+ def events_buffer channel = nil
10
+ @event_buffers ||= MessageBuffer.new channel, :events
11
+ end
12
+ end
13
+
14
+ include Euston::RabbitMq::Exchanges
15
+ include Euston::RabbitMq::Queues
16
+ include Hollywood
17
+
18
+ def initialize channel, exchange_name
19
+
20
+ @channel = channel
21
+ @exchange = get_exchange channel, exchange_name
22
+ @read_model = Euston::RabbitMq::ReadModel::MessageBuffer.send exchange_name
23
+
24
+ @queue = get_queue channel, "#{exchange_name}_buffer"
25
+ @queue.bind @exchange, :routing_key => "#{exchange_name}.#"
26
+
27
+ @queue.when(:message_decode_failed => method(:log_failure),
28
+ :message_failed => method(:handle_failure),
29
+ :message_received => method(:remove_message_from_buffer))
30
+
31
+ @queue.safe_subscribe
32
+ end
33
+
34
+ def handle_failure(message, error, header)
35
+ log_failure message, error
36
+ header.ack
37
+ end
38
+
39
+ def dispatch_due_messages
40
+ dispatched = false
41
+
42
+ @read_model.find_due_messages.to_a.each do |message|
43
+ @read_model.set_next_attempt message
44
+ @exchange.publish message['json'], default_publish_options.merge(:routing_key => "#{@exchange.name}.#{message['type']}")
45
+ dispatched = true
46
+ end
47
+
48
+ callback(:no_messages_were_due) unless dispatched
49
+ end
50
+
51
+ def log_failure message, error
52
+ text = "A buffer queue subscription failed. [Error] #{error.message} [Payload] #{message}"
53
+ err = Euston::RabbitMq::MessageDecodeFailedError.new text
54
+ err.set_backtrace error.backtrace
55
+
56
+ Safely.report! err
57
+ end
58
+
59
+ def push message
60
+ @read_model.buffer_new_message message
61
+ end
62
+
63
+ def remove_message_from_buffer message
64
+ @read_model.remove_published_message message[:headers][:id]
65
+ end
66
+ end
67
+ end
68
+ end
@@ -0,0 +1,50 @@
1
+ module Euston
2
+ module RabbitMq
3
+ class MessageLogger
4
+ class << self
5
+ def commands_logger channel = nil
6
+ @command_logger ||= MessageLogger.new channel, :commands
7
+ end
8
+
9
+ def events_logger channel = nil
10
+ @event_logger ||= MessageLogger.new channel, :events
11
+ end
12
+ end
13
+
14
+ include Euston::RabbitMq::Exchanges
15
+ include Euston::RabbitMq::Queues
16
+
17
+ def initialize channel, exchange_name
18
+ @channel = channel
19
+ @exchange = get_exchange channel, exchange_name
20
+ @read_model = Euston::RabbitMq::ReadModel::MessageLog.send exchange_name
21
+
22
+ @queue = get_queue channel, "#{exchange_name}_log"
23
+ @queue.bind @exchange, :routing_key => "#{exchange_name}.#"
24
+
25
+ @queue.when(:message_decode_failed => method(:log_failure),
26
+ :message_failed => method(:handle_failure),
27
+ :message_received => method(:write_message_to_log))
28
+
29
+ @queue.safe_subscribe
30
+ end
31
+
32
+ def handle_failure(message, error, header)
33
+ log_failure message, error
34
+ header.ack
35
+ end
36
+
37
+ def log_failure message, error
38
+ text = "A log queue subscription failed. [Error] #{error.message} [Payload] #{message}"
39
+ err = Euston::RabbitMq::MessageDecodeFailedError.new text
40
+ err.set_backtrace error.backtrace
41
+
42
+ Safely.report! err
43
+ end
44
+
45
+ def write_message_to_log message
46
+ @read_model.log_new_message message
47
+ end
48
+ end
49
+ end
50
+ end
@@ -0,0 +1,30 @@
1
+ module AMQP
2
+ class Queue
3
+ include Hollywood
4
+
5
+ def safe_subscribe
6
+ cb = Hollywood.instance_method(:callback).bind self
7
+
8
+ self.subscribe :ack => true do |header, body|
9
+ rt = ::RobustThread.new(:args => [header, body], :label => "#{self.name} queue subscriber") do |header, body|
10
+ begin
11
+ message = ::ActiveSupport::JSON.decode body
12
+ message.recursive_symbolize_keys!
13
+
14
+ begin
15
+ cb.call :message_received, message
16
+ header.ack
17
+ rescue Euston::EventStore::ConcurrencyError
18
+ header.reject :requeue => true
19
+ rescue StandardError => e
20
+ cb.call :message_failed, message, e, header
21
+ end
22
+ rescue StandardError => e
23
+ cb.call :message_decode_failed, body, e
24
+ header.ack
25
+ end
26
+ end
27
+ end
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,13 @@
1
+ module Euston
2
+ module RabbitMq
3
+ module Queues
4
+ def default_queue_options
5
+ { :durable => true, :nowait => true }
6
+ end
7
+
8
+ def get_queue channel, name, opts = {}
9
+ channel.queue name.to_s, default_queue_options.merge(opts)
10
+ end
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,36 @@
1
+ module Euston
2
+ module RabbitMq
3
+ module ReadModel
4
+ class FailedMessage
5
+ def initialize
6
+ name = 'failed_messages'
7
+
8
+ mongodb = Euston::RabbitMq.event_store_mongodb
9
+ mongodb.create_collection name unless mongodb.collection_names.include? name
10
+
11
+ @collection = mongodb.collection name
12
+ @collection.ensure_index [ ['failure_timestamp', Mongo::ASCENDING] ], :unique => false, :name => 'failed_messages_failure_timestamp_index'
13
+ end
14
+
15
+ def get_by_id id
16
+ @collection.find_one({ '_id' => id })
17
+ end
18
+
19
+ def find_all
20
+ @collection.find({}, { :sort => [ 'failure_timestamp', Mongo::DESCENDING ] })
21
+ end
22
+
23
+ def log_failure failure
24
+ failure.recursive_stringify_keys!
25
+ failure['_id'] = failure.delete 'message_id'
26
+
27
+ @collection.save(failure, :safe => { :fsync => true })
28
+ end
29
+
30
+ def remove_by_id id
31
+ @collection.remove({ '_id' => id }, :safe => { :fsync => true })
32
+ end
33
+ end
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,52 @@
1
+ module Euston
2
+ module RabbitMq
3
+ module ReadModel
4
+ class MessageBuffer
5
+ class << self
6
+ def commands
7
+ @commands ||= MessageBuffer.new "buffered_commands"
8
+ end
9
+
10
+ def events
11
+ @events ||= MessageBuffer.new "buffered_events"
12
+ end
13
+ end
14
+
15
+ def initialize name
16
+ mongodb = Euston::RabbitMq.event_store_mongodb
17
+ mongodb.create_collection name unless mongodb.collection_names.include? name
18
+
19
+ @collection = mongodb.collection name
20
+ @collection.ensure_index [ ['next_attempt', Mongo::ASCENDING] ], :unique => false, :name => "#{name}_next_attempt_index"
21
+ end
22
+
23
+ def buffer_new_message message
24
+ @collection.save({ '_id' => message[:headers][:id],
25
+ 'type' => message[:headers][:type],
26
+ 'next_attempt' => Time.now.to_f,
27
+ 'json' => ::ActiveSupport::JSON.encode(message) }, :safe => { :fsync => true })
28
+ end
29
+
30
+ def find_due_messages
31
+ query = { 'next_attempt' => { '$lte' => Time.now.to_f } }
32
+ order = [ 'next_attempt', Mongo::ASCENDING ]
33
+
34
+ @collection.find(query).sort(order)
35
+ end
36
+
37
+ def get_by_id id
38
+ @collection.find_one({ '_id' => id })
39
+ end
40
+
41
+ def set_next_attempt message
42
+ @collection.update({ '_id' => message['_id'] },
43
+ { '$set' => { 'next_attempt' => message['next_attempt'] + 10 } }, :safe => { :fsync => true })
44
+ end
45
+
46
+ def remove_published_message id
47
+ @collection.remove({ '_id' => id }, :safe => { :fsync => true })
48
+ end
49
+ end
50
+ end
51
+ end
52
+ end
@@ -0,0 +1,37 @@
1
+ module Euston
2
+ module RabbitMq
3
+ module ReadModel
4
+ class MessageLog
5
+ class << self
6
+ def commands
7
+ @commands ||= MessageLog.new "command_log"
8
+ end
9
+
10
+ def events
11
+ @events ||= MessageLog.new "event_log"
12
+ end
13
+ end
14
+
15
+ def initialize name
16
+ mongodb = Euston::RabbitMq.event_store_mongodb
17
+ mongodb.create_collection name unless mongodb.collection_names.include? name
18
+
19
+ @collection = mongodb.collection name
20
+ @collection.ensure_index [ ['timestamp', Mongo::ASCENDING] ], :unique => false, :name => "#{name}_timestamp_index"
21
+ end
22
+
23
+ def find_all
24
+ @collection.find({}, { :sort => [ 'timestamp', Mongo::DESCENDING ] })
25
+ end
26
+
27
+ def log_new_message message
28
+ @collection.save({ '_id' => message[:headers][:id],
29
+ 'type' => message[:headers][:type],
30
+ 'version' => message[:headers][:version],
31
+ 'timestamp' => Time.now.to_f,
32
+ 'json' => ::ActiveSupport::JSON.encode(message) }, :safe => { :fsync => true })
33
+ end
34
+ end
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,5 @@
1
+ module Euston
2
+ module RabbitMq
3
+ VERSION = "1.0.0"
4
+ end
5
+ end
@@ -0,0 +1,28 @@
1
+ require 'require_all'
2
+ require 'active_support/core_ext/hash/keys'
3
+ require 'active_support/core_ext/string/inflections'
4
+ require 'active_support/json'
5
+ require 'active_support/ordered_hash'
6
+ require 'robustthread'
7
+
8
+ if RUBY_PLATFORM == 'java'
9
+ require 'jessica'
10
+ else
11
+ require 'eventmachine'
12
+ require 'amqp'
13
+ end
14
+
15
+ require 'hash-keys'
16
+ require 'hollywood'
17
+ require 'euston'
18
+ require 'euston-eventstore'
19
+
20
+ require_rel 'euston-rabbitmq'
21
+
22
+ module Euston
23
+ module RabbitMq
24
+ class << self
25
+ attr_accessor :event_store, :event_store_mongodb, :read_model_mongodb
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,69 @@
1
+ require File.expand_path('../spec_helper.rb', __FILE__)
2
+
3
+ unless RUBY_PLATFORM == 'java'
4
+ describe 'command buffer' do
5
+ include EustonRmqSpec
6
+
7
+ context 'the command buffer is created' do
8
+ include Euston::RabbitMq::Queues
9
+
10
+ amqp_before do
11
+ @buffer = Euston::RabbitMq::MessageBuffer.new @channel, :commands
12
+ @queue = get_queue @channel, :commands_buffer
13
+ end
14
+
15
+ it 'declares the commands buffer queue' do
16
+ done(0.5) {
17
+ @queue.name.should == 'commands_buffer'
18
+ @queue.opts.should include(:durable => true)
19
+ }
20
+ end
21
+
22
+ it 'binds the command buffer queue to the commands exchange with a wildcard routing key' do
23
+ received = []
24
+ data = Factory.command
25
+
26
+ delayed(1) {
27
+ exchange = @channel.find_exchange 'commands'
28
+ @queue.when(:message_received => ->(message) { received << message })
29
+ exchange.publish ::ActiveSupport::JSON.encode(data), :routing_key => "commands.#{Euston.uuid.generate}"
30
+ }
31
+
32
+ done(2) {
33
+ received.should have(1).item
34
+ received.first.should == data
35
+ }
36
+ end
37
+ end
38
+
39
+ context 'a command is pushed into the buffer by client code' do
40
+ amqp_before :each do
41
+ @buffer = Euston::RabbitMq::MessageBuffer.new @channel, :commands
42
+ @command = Factory.command
43
+ @buffer.push @command
44
+ @read_model = Euston::RabbitMq::ReadModel::MessageBuffer.new 'buffered_commands'
45
+ end
46
+
47
+ it 'writes the command to persistent storage' do
48
+ @read_model.find_due_messages.to_a.should have(1).items
49
+
50
+ due_command = @read_model.find_due_messages.first
51
+ due_command['_id'].should == @command[:headers][:id]
52
+ due_command['type'].should == @command[:headers][:type]
53
+ due_command['json'].should == ::ActiveSupport::JSON.encode(@command)
54
+
55
+ done(0.3)
56
+ end
57
+
58
+ it 'removes the command from the buffer storage when it is obtained from the buffer queue' do
59
+ delayed(1) {
60
+ @buffer.dispatch_due_messages
61
+ }
62
+
63
+ done(3) {
64
+ @read_model.get_by_id(@command[:headers][:id]).should be_nil
65
+ }
66
+ end
67
+ end
68
+ end
69
+ end
@@ -0,0 +1,69 @@
1
+ require File.expand_path('../spec_helper.rb', __FILE__)
2
+
3
+ unless RUBY_PLATFORM == 'java'
4
+ describe 'event buffer' do
5
+ include EustonRmqSpec
6
+
7
+ context 'the event buffer is created' do
8
+ include Euston::RabbitMq::Queues
9
+
10
+ amqp_before do
11
+ @buffer = Euston::RabbitMq::MessageBuffer.new @channel, :events
12
+ @queue = get_queue @channel, :events_buffer
13
+ end
14
+
15
+ it 'declares the events buffer queue' do
16
+ done(0.5) {
17
+ @queue.name.should == 'events_buffer'
18
+ @queue.opts.should include(:durable => true)
19
+ }
20
+ end
21
+
22
+ it 'binds the event buffer queue to the events exchange with a wildcard routing key' do
23
+ received = []
24
+ data = Factory.command
25
+
26
+ delayed(0.3) {
27
+ exchange = @channel.find_exchange 'events'
28
+ @queue.when(:message_received => ->(message) { received << message })
29
+ exchange.publish ::ActiveSupport::JSON.encode(data), :routing_key => "events.#{Euston.uuid.generate}"
30
+ }
31
+
32
+ done(2) {
33
+ received.should have(1).item
34
+ received.first.should == data
35
+ }
36
+ end
37
+ end
38
+
39
+ context 'an event is pushed into the buffer by client code' do
40
+ amqp_before :each do
41
+ @buffer = Euston::RabbitMq::MessageBuffer.new @channel, :events
42
+ @command = Factory.command
43
+ @buffer.push @command
44
+ @read_model = Euston::RabbitMq::ReadModel::MessageBuffer.new 'buffered_events'
45
+ end
46
+
47
+ it 'writes the event to persistent storage' do
48
+ @read_model.find_due_messages.to_a.should have(1).items
49
+
50
+ due_command = @read_model.find_due_messages.first
51
+ due_command['_id'].should == @command[:headers][:id]
52
+ due_command['type'].should == @command[:headers][:type]
53
+ due_command['json'].should == ::ActiveSupport::JSON.encode(@command)
54
+
55
+ done(0.3)
56
+ end
57
+
58
+ it 'removes the event from the buffer storage when it is obtained from the buffer queue' do
59
+ delayed(1) {
60
+ @buffer.dispatch_due_messages
61
+ }
62
+
63
+ done(3) {
64
+ @read_model.get_by_id(@command[:headers][:id]).should be_nil
65
+ }
66
+ end
67
+ end
68
+ end
69
+ end
@@ -0,0 +1,28 @@
1
+ require File.expand_path('../spec_helper.rb', __FILE__)
2
+
3
+ unless RUBY_PLATFORM == 'java'
4
+ describe 'exchange declaration' do
5
+ include EustonRmqSpec
6
+
7
+ context 'declaring the commands exchange' do
8
+ include Euston::RabbitMq::Exchanges
9
+ it 'declares the commands exchange' do
10
+ exchange = get_exchange @channel, :commands
11
+
12
+ done(0.3) {
13
+ exchange.name.should == 'commands'
14
+ exchange.should be_durable
15
+ }
16
+ end
17
+
18
+ it 'declares the events exchange' do
19
+ exchange = get_exchange @channel, :events
20
+
21
+ done(0.3) {
22
+ exchange.name.should == 'events'
23
+ exchange.should be_durable
24
+ }
25
+ end
26
+ end
27
+ end
28
+ end