euston-daemons 1.0.5 → 1.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. data/Gemfile +4 -1
  2. data/Rakefile +2 -3
  3. data/euston-daemons.gemspec +50 -39
  4. data/lib/euston-daemons.rb +14 -1
  5. data/lib/euston-daemons/euston/daemon.rb +99 -0
  6. data/lib/euston-daemons/euston/daemon_component.rb +25 -0
  7. data/lib/euston-daemons/euston/daemon_component_host.rb +66 -0
  8. data/lib/euston-daemons/euston/daemon_environment.rb +59 -0
  9. data/lib/euston-daemons/euston/exceptions.rb +9 -0
  10. data/lib/euston-daemons/euston/stopwatch.rb +15 -0
  11. data/lib/euston-daemons/pipeline/config/environment.rb +78 -0
  12. data/lib/euston-daemons/pipeline/lib/command_logger/component.rb +54 -0
  13. data/lib/euston-daemons/pipeline/lib/command_logger/log.rb +31 -0
  14. data/lib/euston-daemons/pipeline/lib/command_processor/component.rb +50 -0
  15. data/lib/euston-daemons/pipeline/lib/command_processor/default_commands/retry_failed_message.rb +13 -0
  16. data/lib/euston-daemons/pipeline/lib/command_processor/default_handlers/retry_failed_message.rb +34 -0
  17. data/lib/euston-daemons/pipeline/lib/command_processor/failed_message.rb +36 -0
  18. data/lib/euston-daemons/pipeline/lib/daemon.rb +85 -0
  19. data/lib/euston-daemons/pipeline/lib/event_processor/component.rb +67 -0
  20. data/lib/euston-daemons/pipeline/lib/event_processor/default_handlers/message_failure.rb +30 -0
  21. data/lib/euston-daemons/pipeline/lib/event_store_dispatcher/component.rb +68 -0
  22. data/lib/euston-daemons/pipeline/lib/message_buffer/buffer.rb +85 -0
  23. data/lib/euston-daemons/pipeline/lib/message_buffer/component.rb +59 -0
  24. data/lib/euston-daemons/pipeline/lib/snapshotter/component.rb +48 -0
  25. data/lib/euston-daemons/pipeline/rake_task.rb +49 -0
  26. data/lib/euston-daemons/rake_task.rb +63 -66
  27. data/lib/euston-daemons/rake_tasks.rb +3 -5
  28. data/lib/euston-daemons/version.rb +1 -1
  29. data/spec/daemons/command_processor_spec.rb +48 -0
  30. data/spec/daemons/event_processor_spec.rb +55 -0
  31. data/spec/daemons/message_buffer_spec.rb +106 -0
  32. data/spec/daemons/snapshotter_spec.rb +96 -0
  33. data/spec/spec_helper.rb +91 -0
  34. data/spec/support/factories/commands.rb +16 -0
  35. data/spec/support/factories/commit.rb +7 -0
  36. data/spec/support/factories/event_message.rb +12 -0
  37. data/spec/support/factories/events.rb +8 -0
  38. data/spec/support/filters.rb +13 -0
  39. data/spec/support/sample_model/commands.rb +14 -0
  40. data/spec/support/sample_model/counter.rb +36 -0
  41. data/spec/support/sample_model/counter2.rb +46 -0
  42. data/spec/support/stub_retrying_subscription.rb +9 -0
  43. metadata +131 -67
  44. data/lib/euston-daemons/command_processor_daemon/config/environment.rb +0 -25
  45. data/lib/euston-daemons/command_processor_daemon/lib/components/command_handler_component.rb +0 -56
  46. data/lib/euston-daemons/command_processor_daemon/lib/daemon.rb +0 -43
  47. data/lib/euston-daemons/command_processor_daemon/lib/settings.rb +0 -22
  48. data/lib/euston-daemons/command_processor_daemon/rake_task.rb +0 -34
  49. data/lib/euston-daemons/event_processor_daemon/config/environment.rb +0 -25
  50. data/lib/euston-daemons/event_processor_daemon/lib/components/event_handler_component.rb +0 -58
  51. data/lib/euston-daemons/event_processor_daemon/lib/daemon.rb +0 -71
  52. data/lib/euston-daemons/event_processor_daemon/lib/settings.rb +0 -26
  53. data/lib/euston-daemons/event_processor_daemon/rake_task.rb +0 -37
  54. data/lib/euston-daemons/framework/basic_component.rb +0 -33
  55. data/lib/euston-daemons/framework/channel_thread.rb +0 -22
  56. data/lib/euston-daemons/framework/component_shutdown.rb +0 -22
  57. data/lib/euston-daemons/framework/daemon.rb +0 -27
  58. data/lib/euston-daemons/framework/handler_bindings_component.rb +0 -56
  59. data/lib/euston-daemons/framework/queue.rb +0 -71
  60. data/lib/euston-daemons/message_buffer_daemon/config/environment.rb +0 -28
  61. data/lib/euston-daemons/message_buffer_daemon/lib/components/buffer_component.rb +0 -73
  62. data/lib/euston-daemons/message_buffer_daemon/lib/components/event_store_component.rb +0 -52
  63. data/lib/euston-daemons/message_buffer_daemon/lib/daemon.rb +0 -48
  64. data/lib/euston-daemons/message_buffer_daemon/lib/message_logger.rb +0 -54
  65. data/lib/euston-daemons/message_buffer_daemon/lib/publisher.rb +0 -56
  66. data/lib/euston-daemons/message_buffer_daemon/lib/read_model/message_log.rb +0 -36
  67. data/lib/euston-daemons/message_buffer_daemon/lib/settings.rb +0 -14
  68. data/lib/euston-daemons/message_buffer_daemon/lib/subscriber.rb +0 -60
  69. data/lib/euston-daemons/message_buffer_daemon/rake_task.rb +0 -30
@@ -1,7 +1,5 @@
1
1
  require 'euston-daemons'
2
- require 'rake/tasklib'
3
2
 
4
- require_rel 'rake_task.rb'
5
- require_rel 'command_processor_daemon/rake_task.rb'
6
- require_rel 'event_processor_daemon/rake_task.rb'
7
- require_rel 'message_buffer_daemon/rake_task.rb'
3
+ require 'rake/tasklib'
4
+ require 'euston-daemons/rake_task'
5
+ require 'euston-daemons/pipeline/rake_task'
@@ -1,5 +1,5 @@
1
1
  module Euston
2
2
  module Daemons
3
- VERSION = "1.0.5"
3
+ VERSION = "1.2.1"
4
4
  end
5
5
  end
@@ -0,0 +1,48 @@
1
+ describe 'command handler component', :purge_event_store, :purge_rabbitmq do
2
+ require 'euston-daemons/pipeline/lib/command_processor/component'
3
+
4
+ let(:logger) { Euston::NullLogger.instance }
5
+ let(:handlers) { [] }
6
+ let(:subscription) { StubRetryingSubscription.new }
7
+ let(:client) { Euston::Daemons::Pipeline::CommandProcessor::Component.new @channel, handlers, 1, logger }
8
+ let(:aggregate) { Euston::Daemons::SampleModel::Counter2.new }
9
+ let(:saved_aggregates) { [] }
10
+
11
+ before do
12
+ Euston::RabbitMq::RetryingSubscription.stub(:new) { subscription }
13
+ Euston::Repository.stub(:find) { aggregate }
14
+ Euston::Repository.stub(:save) { |aggregate| saved_aggregates << aggregate }
15
+ end
16
+
17
+ context 'a message is received' do
18
+ before do
19
+ client.run
20
+ command = Factory.build(:create_counter_command, :id => Uuid.generate).to_hash
21
+ subscription.stub_message_receipt command
22
+ end
23
+
24
+ context 'the message is not explicitly handled by a command handler class' do
25
+ subject { saved_aggregates }
26
+ it { should_not be_empty }
27
+ end
28
+
29
+ context 'the message is explicitly handled by a command handler class' do
30
+ class CreateCounter
31
+ include Euston::CommandHandler
32
+
33
+ class << self
34
+ attr_accessor :was_called
35
+ end
36
+
37
+ version 1 do |headers, command|
38
+ self.class.was_called = true
39
+ end
40
+ end
41
+
42
+ let(:handlers) { [ Euston::RabbitMq::HandlerReference.new(Object, CreateCounter, :CreateCounter) ] }
43
+
44
+ subject { CreateCounter.was_called }
45
+ it { should be_true }
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,55 @@
1
+ describe 'event handler client', :purge_event_store, :purge_rabbitmq do
2
+ require 'euston-daemons/pipeline/lib/event_processor/component'
3
+
4
+ let(:logger) { Euston::NullLogger.instance }
5
+ let(:subscription) { StubRetryingSubscription.new }
6
+ let(:client) { Euston::Daemons::Pipeline::EventProcessor::Component.new @channel, reference, 1, logger }
7
+ let(:aggregate) { Euston::Daemons::SampleModel::Counter2.new }
8
+ let(:saved_aggregates) { [] }
9
+
10
+ before do
11
+ Euston::RabbitMq::RetryingSubscription.stub(:new) { subscription }
12
+ Euston::Repository.stub(:find) { aggregate }
13
+ Euston::Repository.stub(:save) { |aggregate| saved_aggregates << aggregate }
14
+ end
15
+
16
+ context 'a message is received' do
17
+ before do
18
+ client.run
19
+
20
+ event = { :headers => { :id => Uuid.generate,
21
+ :type => :external_thing_happened,
22
+ :timestamp => Time.now.to_f,
23
+ :version => 1 },
24
+ :body => {} }
25
+
26
+ subscription.stub_message_receipt event
27
+ end
28
+
29
+ context 'the event handler is bound to an aggregate root' do
30
+ let(:reference) { Euston::RabbitMq::HandlerReference.new(Euston::Daemons::SampleModel, Euston::Daemons::SampleModel::Counter2, :Counter2) }
31
+
32
+ subject { saved_aggregates }
33
+ it { should_not be_empty }
34
+ end
35
+
36
+ context 'the message is explicitly handled by an event handler class' do
37
+ class ExternalThingHappenedListener
38
+ include Euston::EventHandler
39
+
40
+ class << self
41
+ attr_accessor :was_called
42
+ end
43
+
44
+ subscribes :external_thing_happened, do |headers, command|
45
+ self.class.was_called = true
46
+ end
47
+ end
48
+
49
+ let(:reference) { Euston::RabbitMq::HandlerReference.new(Object, ExternalThingHappenedListener, :ExternalThingHappenedListener) }
50
+
51
+ subject { ExternalThingHappenedListener.was_called }
52
+ it { should be_true }
53
+ end
54
+ end
55
+ end
@@ -0,0 +1,106 @@
1
+ describe 'message buffer component', :purge_event_store do
2
+ require 'euston-daemons/pipeline/lib/message_buffer/buffer'
3
+ require 'euston-daemons/pipeline/lib/message_buffer/component'
4
+
5
+ let(:logger) { Euston::NullLogger.instance }
6
+ let(:channel) { double('channel').as_null_object }
7
+ let(:command) { Factory.build(:create_counter_command, :id => Uuid.generate).to_hash }
8
+ let(:exchange) { double('exchange').as_null_object }
9
+ let(:published) { [] }
10
+ let(:collection) { @event_store_database[buffer.name] }
11
+ let(:buffer) { Euston::Daemons::Pipeline::MessageBuffer::Buffer.new @event_store_database }
12
+ let(:component) { Euston::Daemons::Pipeline::MessageBuffer::Component.new channel }
13
+
14
+ module Euston
15
+ class DaemonEnvironment
16
+ class << self
17
+ attr_accessor :event_store_mongodb
18
+ end
19
+ end
20
+ end
21
+
22
+ before do
23
+ Euston::DaemonEnvironment.event_store_mongodb = @event_store_database
24
+ channel.stub(:topic) { exchange }
25
+ exchange.stub(:name) { 'commands' }
26
+ exchange.stub(:publish) { |json, options| published << { :json => json, :options => options } }
27
+ end
28
+
29
+ context 'there are no commands in the buffer' do
30
+ describe 'command buffer size' do
31
+ subject { collection.count }
32
+ it { should == 0 }
33
+ end
34
+
35
+ context 'the component runs' do
36
+ before { component.run }
37
+
38
+ describe 'published commands' do
39
+ subject { published }
40
+ it { should be_empty }
41
+ end
42
+ end
43
+ end
44
+
45
+ context 'there is a due command' do
46
+
47
+ before do
48
+ buffer.enqueue :commands, command
49
+ end
50
+
51
+ describe 'command buffer size' do
52
+ subject { collection.count }
53
+ it { should == 1 }
54
+ end
55
+
56
+ context 'the component runs' do
57
+ before { component.run }
58
+
59
+ describe 'published commands' do
60
+ subject { published }
61
+ it { should have(1).item }
62
+ end
63
+
64
+ describe 'published command' do
65
+ subject { published.first }
66
+ its([:json]) { should == command.to_json }
67
+ end
68
+
69
+ describe 'buffered command' do
70
+ subject { collection.count }
71
+ it { should == 0 }
72
+ end
73
+ end
74
+ end
75
+
76
+ context 'there is a command that is not yet due' do
77
+ before do
78
+ buffer.enqueue :commands, command
79
+ collection.update({}, { '$set' => { 'dispatch_at' => Time.now.to_f + 1000 } }, { :safe => true })
80
+ end
81
+
82
+ context 'the component runs' do
83
+ before { component.run }
84
+
85
+ describe 'published commands' do
86
+ subject { published }
87
+ it { should be_empty }
88
+ end
89
+ end
90
+ end
91
+
92
+ context 'a command is buffered for dispatch in the future' do
93
+ before do
94
+ buffer.enqueue :commands, command, Time.now.to_f + 1000
95
+ end
96
+
97
+ context 'the component runs' do
98
+ before { component.run }
99
+
100
+ describe 'published commands' do
101
+ subject { published }
102
+ it { should be_empty }
103
+ end
104
+ end
105
+ end
106
+ end
@@ -0,0 +1,96 @@
1
+ describe 'snapshotter component', :purge_event_store, :euston_event_store do
2
+ require 'euston-daemons/pipeline/lib/snapshotter/component'
3
+
4
+ let(:logger) { Euston::NullLogger.instance }
5
+ let(:threshold) { 2 }
6
+ let(:snapshotter) { Euston::Daemons::Pipeline::Snapshotter::Component.new @event_store, threshold, logger }
7
+ let(:commands) { [] }
8
+ let(:counter_id) { Uuid.generate }
9
+
10
+ def publish_commands_then_take_snapshot commands
11
+ commands.each { |c| publish_command c }
12
+ sleep 0.25 # to allow the async stream head update to finish
13
+ snapshotter.run
14
+ end
15
+
16
+ before { publish_commands_then_take_snapshot commands }
17
+
18
+ context 'there are no streams' do
19
+ describe 'snapshots collection size' do
20
+ subject { @event_store_database['snapshots'].count }
21
+ it { should == 0 }
22
+ end
23
+ end
24
+
25
+ context 'there is a stream that is too small to snapshot' do
26
+ let(:commands) { [ Factory.build(:create_counter_command) ] }
27
+
28
+ describe 'snapshots collection size' do
29
+ subject { @event_store_database['snapshots'].count }
30
+ it { should == 0 }
31
+ end
32
+ end
33
+
34
+ context 'there is a stream that is large enough to snapshot' do
35
+ let(:commands) { [ Factory.build(:create_counter_command, :id => counter_id),
36
+ Factory.build(:increment_counter_command, :counter_id => counter_id, :amount => 1),
37
+ Factory.build(:increment_counter_command, :counter_id => counter_id, :amount => 2) ] }
38
+
39
+ describe 'snapshots collection size' do
40
+ subject { @event_store_database['snapshots'].count }
41
+ it { should == 1 }
42
+ end
43
+
44
+ describe 'snapshot' do
45
+ subject { @event_store.get_snapshot counter_id }
46
+ its(:headers) { should == { :version => 1 } }
47
+ its(:payload) { should == { :count => 3 } }
48
+ end
49
+
50
+ describe 'reloading from the snapshot' do
51
+ subject { Euston::Repository.find Euston::Daemons::SampleModel::Counter, counter_id }
52
+ its(:count) { should == 3 }
53
+ its(:snapshot_loaded) { should == { :count => 3 }}
54
+ end
55
+ end
56
+
57
+ context 'multiple snapshots taken over time' do
58
+ let(:commands) { [ Factory.build(:create_counter_command, :id => counter_id),
59
+ Factory.build(:increment_counter_command, :counter_id => counter_id, :amount => 1),
60
+ Factory.build(:increment_counter_command, :counter_id => counter_id, :amount => 2) ] }
61
+ let(:commands2) { [ Factory.build(:increment_counter_command, :counter_id => counter_id, :amount => 3),
62
+ Factory.build(:increment_counter_command, :counter_id => counter_id, :amount => 5),
63
+ Factory.build(:increment_counter_command, :counter_id => counter_id, :amount => 8) ] }
64
+ let(:commands3) { [ Factory.build(:increment_counter_command, :counter_id => counter_id, :amount => 13),
65
+ Factory.build(:increment_counter_command, :counter_id => counter_id, :amount => 21),
66
+ Factory.build(:increment_counter_command, :counter_id => counter_id, :amount => 34) ] }
67
+
68
+ before do
69
+ publish_commands_then_take_snapshot commands2
70
+ publish_commands_then_take_snapshot commands3
71
+ end
72
+
73
+ describe 'snapshots collection size' do
74
+ subject { @event_store_database['snapshots'].count }
75
+ it { should == 3 }
76
+ end
77
+
78
+ describe 'first snapshot' do
79
+ subject { @event_store.get_snapshot counter_id, 3 }
80
+ its(:headers) { should == { :version => 1 } }
81
+ its(:payload) { should == { :count => 3 } }
82
+ end
83
+
84
+ describe 'second snapshot' do
85
+ subject { @event_store.get_snapshot counter_id, 6 }
86
+ its(:headers) { should == { :version => 1 } }
87
+ its(:payload) { should == { :count => 19 } }
88
+ end
89
+
90
+ describe 'third snapshot' do
91
+ subject { @event_store.get_snapshot counter_id }
92
+ its(:headers) { should == { :version => 1 } }
93
+ its(:payload) { should == { :count => 87 } }
94
+ end
95
+ end
96
+ end
@@ -0,0 +1,91 @@
1
+ jruby = RUBY_PLATFORM.to_s == 'java'
2
+
3
+ require 'ap'
4
+ require 'rabbitmqadmin-cli'
5
+ require 'euston'
6
+ require 'euston-rabbitmq'
7
+ require 'euston-eventstore'
8
+ require 'euston-daemons'
9
+ require 'cranky'
10
+ require 'ffaker'
11
+ require 'mongo' unless jruby
12
+ require 'jmongo' if jruby
13
+ require 'hot_bunnies' if jruby
14
+ require 'safely'
15
+
16
+ specs_path = File.dirname __FILE__
17
+ require File.join specs_path, 'support', 'factories', 'event_message'
18
+ require File.join specs_path, 'support', 'factories', 'commit'
19
+ require File.join specs_path, 'support', 'factories', 'commands'
20
+ require File.join specs_path, 'support', 'filters'
21
+ require File.join specs_path, 'support', 'stub_retrying_subscription'
22
+ require File.join specs_path, 'support', 'sample_model', 'commands'
23
+ require File.join specs_path, 'support', 'sample_model', 'counter'
24
+ require File.join specs_path, 'support', 'sample_model', 'counter2'
25
+
26
+ Safely.configure do |config|
27
+ config.strategies = [ Safely::Strategy::Log ]
28
+ end
29
+
30
+ amqp_config = { :host => 'localhost',
31
+ :virtual_host => 'euston-daemons-test',
32
+ :username => 'euston-daemons-test-user',
33
+ :password => 'password' }
34
+
35
+ mongo_connection = Mongo::Connection.from_uri 'mongodb://0.0.0.0:27017/?safe=true;fsync=true;autoconnectretry=true;w=1;'
36
+ event_store_database = Mongo::DB.new 'euston-daemons-test-event-store', mongo_connection
37
+ projections_database = Mongo::DB.new 'euston-daemons-test-projections', mongo_connection
38
+
39
+ RSpec.configure do |config|
40
+ config.treat_symbols_as_metadata_keys_with_true_values = true
41
+
42
+ config.before(:each) do
43
+ @event_store_database = event_store_database
44
+ @projections_database = projections_database
45
+ end
46
+
47
+ config.before(:each, :euston_event_store) do
48
+ mongo_config = Euston::EventStore::Persistence::Mongodb::Config.instance
49
+ mongo_config.database = event_store_database.name
50
+
51
+ @persistence_factory = Euston::EventStore::Persistence::Mongodb::MongoPersistenceFactory.build
52
+ @persistence_factory.init
53
+
54
+ @event_store = Euston::EventStore::OptimisticEventStore.new @persistence_factory
55
+ Euston::Repository.event_store = @event_store
56
+
57
+ def publish_command command
58
+ hash = command.to_hash
59
+ command = { :headers => Euston::CommandHeaders.from_hash(hash[:headers]), :body => hash[:body] }
60
+ Euston::CommandBus.publish command[:headers], command[:body]
61
+ end
62
+ end
63
+
64
+ config.before(:each, :purge_event_store) do
65
+ purge_database event_store_database
66
+ end
67
+
68
+ config.before(:each, :purge_mongo) do
69
+ purge_database event_store_database
70
+ purge_database projections_database
71
+ end
72
+
73
+ config.before(:each, :purge_rabbitmq) do
74
+ initialize_rabbitmq amqp_config
75
+ end
76
+
77
+ config.before(:all, :purge_rabbitmq) do
78
+ @amqp_connection = HotBunnies.connect amqp_config
79
+
80
+ initialize_amqp @amqp_connection
81
+ end
82
+
83
+ config.before(:each, :purge_projections) do
84
+ purge_database projections_database
85
+ end
86
+
87
+ config.after(:all, :purge_rabbitmq) do
88
+ @channel.close
89
+ @amqp_connection.close
90
+ end
91
+ end
@@ -0,0 +1,16 @@
1
+ class Cranky::Factory
2
+ def create_counter_command
3
+ hash = define :class => Hash,
4
+ :id => Uuid.generate
5
+
6
+ Euston::Daemons::SampleModel::CreateCounter.new hash
7
+ end
8
+
9
+ def increment_counter_command
10
+ hash = define :class => Hash,
11
+ :counter_id => Uuid.generate,
12
+ :amount => 1 + rand(9)
13
+
14
+ Euston::Daemons::SampleModel::IncrementCounter.new hash
15
+ end
16
+ end
@@ -0,0 +1,7 @@
1
+ class Cranky::Factory
2
+ def commit
3
+ Euston::EventStore::Commit.new :stream_id => Uuid.generate,
4
+ :commit_id => Uuid.generate,
5
+ :events => (1..(rand(2) + 1)).map { Factory.build :event_message }
6
+ end
7
+ end