euston-daemons 1.0.4-java → 1.1.0-java

Sign up to get free protection for your applications and to get access to all the features.
Files changed (84) hide show
  1. data/Gemfile +3 -1
  2. data/Rakefile +2 -3
  3. data/euston-daemons.gemspec +61 -26
  4. data/lib/euston-daemons/command_processor_daemon/config/environment.rb +27 -18
  5. data/lib/euston-daemons/command_processor_daemon/lib/clients/command_handler.rb +37 -0
  6. data/lib/euston-daemons/command_processor_daemon/lib/command_handlers/retry_failed_message.rb +35 -0
  7. data/lib/euston-daemons/command_processor_daemon/lib/daemon.rb +11 -32
  8. data/lib/euston-daemons/command_processor_daemon/lib/mongo_models/failed_message.rb +34 -0
  9. data/lib/euston-daemons/command_processor_daemon/rake_task.rb +15 -13
  10. data/lib/euston-daemons/euston/daemon.rb +93 -0
  11. data/lib/euston-daemons/euston/daemon_client.rb +24 -0
  12. data/lib/euston-daemons/euston/daemon_component.rb +65 -0
  13. data/lib/euston-daemons/euston/daemon_environment.rb +54 -0
  14. data/lib/euston-daemons/event_processor_daemon/config/environment.rb +27 -16
  15. data/lib/euston-daemons/event_processor_daemon/lib/clients/event_handler.rb +42 -0
  16. data/lib/euston-daemons/event_processor_daemon/lib/daemon.rb +13 -60
  17. data/lib/euston-daemons/event_processor_daemon/lib/event_handlers/message_failure.rb +27 -0
  18. data/lib/euston-daemons/event_processor_daemon/rake_task.rb +18 -16
  19. data/lib/euston-daemons/message_buffer_daemon/config/environment.rb +39 -25
  20. data/lib/euston-daemons/message_buffer_daemon/lib/clients/command_buffer_cleanup.rb +9 -0
  21. data/lib/euston-daemons/message_buffer_daemon/lib/clients/command_buffer_publisher.rb +9 -0
  22. data/lib/euston-daemons/message_buffer_daemon/lib/clients/command_logger.rb +9 -0
  23. data/lib/euston-daemons/message_buffer_daemon/lib/clients/euston_exchange_accessors.rb +15 -0
  24. data/lib/euston-daemons/message_buffer_daemon/lib/clients/event_buffer_cleanup.rb +9 -0
  25. data/lib/euston-daemons/message_buffer_daemon/lib/clients/event_buffer_publisher.rb +9 -0
  26. data/lib/euston-daemons/message_buffer_daemon/lib/clients/event_logger.rb +9 -0
  27. data/lib/euston-daemons/message_buffer_daemon/lib/clients/event_store_dispatcher.rb +25 -0
  28. data/lib/euston-daemons/message_buffer_daemon/lib/clients/message_buffer_cleanup.rb +52 -0
  29. data/lib/euston-daemons/message_buffer_daemon/lib/clients/message_buffer_publisher.rb +37 -0
  30. data/lib/euston-daemons/message_buffer_daemon/lib/clients/message_logger.rb +45 -0
  31. data/lib/euston-daemons/message_buffer_daemon/lib/clients/mongo_model_accessors.rb +21 -0
  32. data/lib/euston-daemons/message_buffer_daemon/lib/daemon.rb +11 -42
  33. data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/command_buffer.rb +11 -0
  34. data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/command_log.rb +11 -0
  35. data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/event_buffer.rb +11 -0
  36. data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/event_log.rb +11 -0
  37. data/lib/euston-daemons/message_buffer_daemon/lib/mongo_models/message_buffer.rb +57 -0
  38. data/lib/euston-daemons/message_buffer_daemon/lib/{read_model → mongo_models}/message_log.rb +4 -11
  39. data/lib/euston-daemons/message_buffer_daemon/rake_task.rb +13 -11
  40. data/lib/euston-daemons/rake_task.rb +41 -65
  41. data/lib/euston-daemons/rake_tasks.rb +5 -5
  42. data/lib/euston-daemons/snapshot_daemon/lib/clients/snapshotter.rb +43 -0
  43. data/lib/euston-daemons/version.rb +1 -1
  44. data/lib/euston-daemons.rb +6 -1
  45. data/sample/Rakefile +63 -0
  46. data/sample/amqp_config.yml +14 -0
  47. data/sample/command_handlers.rb +17 -0
  48. data/sample/command_processor_daemon_config.yml +9 -0
  49. data/sample/event_handlers.rb +21 -0
  50. data/sample/event_processor_daemon_config.yml +8 -0
  51. data/sample/message_buffer_daemon_config.yml +8 -0
  52. data/sample/mongoid_config.yml +13 -0
  53. data/sample/pids/.placeholder +0 -0
  54. data/spec/daemons/command_buffer_publisher_spec.rb +110 -0
  55. data/spec/daemons/command_handler_spec.rb +48 -0
  56. data/spec/daemons/event_handler_spec.rb +55 -0
  57. data/spec/daemons/snapshot_client_spec.rb +98 -0
  58. data/spec/spec_helper.rb +77 -0
  59. data/spec/support/factories/commands.rb +16 -0
  60. data/spec/support/factories/commit.rb +7 -0
  61. data/spec/support/factories/event_message.rb +12 -0
  62. data/spec/support/factories/events.rb +8 -0
  63. data/spec/support/filters.rb +14 -0
  64. data/spec/support/sample_model/commands.rb +14 -0
  65. data/spec/support/sample_model/counter.rb +36 -0
  66. data/spec/support/sample_model/counter2.rb +46 -0
  67. data/spec/support/stub_retrying_subscription.rb +9 -0
  68. metadata +134 -67
  69. data/lib/euston-daemons/command_processor_daemon/lib/components/command_handler_component.rb +0 -56
  70. data/lib/euston-daemons/command_processor_daemon/lib/settings.rb +0 -22
  71. data/lib/euston-daemons/event_processor_daemon/lib/components/event_handler_component.rb +0 -58
  72. data/lib/euston-daemons/event_processor_daemon/lib/settings.rb +0 -26
  73. data/lib/euston-daemons/framework/basic_component.rb +0 -33
  74. data/lib/euston-daemons/framework/channel_thread.rb +0 -22
  75. data/lib/euston-daemons/framework/component_shutdown.rb +0 -22
  76. data/lib/euston-daemons/framework/daemon.rb +0 -27
  77. data/lib/euston-daemons/framework/handler_bindings_component.rb +0 -56
  78. data/lib/euston-daemons/framework/queue.rb +0 -71
  79. data/lib/euston-daemons/message_buffer_daemon/lib/components/buffer_component.rb +0 -73
  80. data/lib/euston-daemons/message_buffer_daemon/lib/components/event_store_component.rb +0 -52
  81. data/lib/euston-daemons/message_buffer_daemon/lib/message_logger.rb +0 -54
  82. data/lib/euston-daemons/message_buffer_daemon/lib/publisher.rb +0 -56
  83. data/lib/euston-daemons/message_buffer_daemon/lib/settings.rb +0 -14
  84. data/lib/euston-daemons/message_buffer_daemon/lib/subscriber.rb +0 -60
@@ -1,71 +0,0 @@
1
- class RabbitMQClient
2
- class Queue
3
- include Hollywood
4
-
5
- attr_writer :timeout
6
-
7
- def delivery_timeout
8
- @timeout ||= 500
9
- end
10
-
11
- def safe_subscribe
12
- _consumer = self.consumer
13
- until Thread.current[:stop] do
14
- safe_subscribe_with_timeout(_consumer, self.delivery_timeout)
15
- end
16
- end
17
-
18
- def safe_handle_message(msg)
19
- begin
20
- message = JSON.parse_sym(msg.body)
21
- begin
22
- callback :message_received, message
23
- msg.ack!
24
- rescue Euston::EventStore::ConcurrencyError
25
- msg.reject! true #requeue
26
- rescue => e
27
- callback :message_failed, message, e, msg
28
- Safely.report! e
29
- end
30
- rescue => e
31
- callback :message_decode_failed, msg.body, e
32
- msg.ack!
33
- Safely.report! e
34
- end
35
- end
36
-
37
- def consumer(auto_ack=false)
38
- consumer = QueueingConsumer.new(@channel)
39
- @channel.basic_consume(@name, auto_ack, consumer)
40
- consumer
41
- end
42
-
43
- def safe_subscribe_with_timeout(consumer, timeout=500)
44
- loop do
45
- delivery = nil
46
- begin
47
- delivery = consumer.next_delivery(timeout)
48
- rescue NativeException => e
49
- Thread.current[:exception] = e
50
- break
51
- end
52
- break if delivery.nil?
53
- remsg = ReactiveMessage.new(@channel, delivery, String.from_java_bytes(delivery.get_body))
54
- safe_handle_message remsg
55
- @channel.basic_ack(delivery.envelope.delivery_tag, false) if remsg.should_acknowledge?
56
- end
57
- end
58
- end
59
-
60
- class ReactiveMessage
61
- def reject(opts={})
62
- self.reject!(opts.fetch(:requeue, true))
63
- end
64
- def ack(multiple=false)
65
- self.ack!
66
- end
67
- def method()
68
- self.envelope
69
- end
70
- end
71
- end
@@ -1,73 +0,0 @@
1
- module Euston
2
- module MessageBufferDaemon
3
- class BufferComponent
4
- include Euston::Daemons::ComponentShutdown
5
-
6
- class << self
7
- def command_component pub_channel, sub_channel
8
- self.new Publisher.commands_buffer(pub_channel), Subscriber.commands_buffer(sub_channel)
9
- end
10
-
11
- def event_component pub_channel, sub_channel
12
- self.new Publisher.events_buffer(pub_channel), Subscriber.events_buffer(sub_channel)
13
- end
14
- end
15
-
16
- attr_reader :pub_thread, :sub_thread
17
- attr_writer :wait_time, :subscribe_timeout
18
-
19
- def initialize pub_buffer, sub_buffer
20
- @pub_buffer = pub_buffer
21
- @sub_buffer = sub_buffer
22
- end
23
-
24
- def wait_time
25
- @wait_time ||= 0.2
26
- end
27
-
28
- def subscribe_timeout
29
- @subscribe_timeout ||= 2000
30
- end
31
-
32
- def thread_state
33
- {:publish => @pub_thread.status, :subscribe => @sub_thread.status}
34
- end
35
-
36
- def stop
37
- @pub_thread[:stop], @sub_thread[:stop] = true, true
38
- @pub_buffer.disconnect
39
- @sub_buffer.disconnect
40
- end
41
-
42
- def start
43
- @pub_thread = Thread.new do
44
- until Thread.current[:stop] do
45
- begin
46
- @msg_dispatched_count = 0
47
- begin
48
- @msg_dispatched_count = @pub_buffer.dispatch_due_messages
49
- check_exception_and_shutdown
50
- end until Thread.current[:stop] || @msg_dispatched_count.zero?
51
- rescue => e
52
- Thread.current[:exception] = e
53
- end
54
- check_exception_and_shutdown
55
- sleep self.wait_time
56
- end
57
- end
58
-
59
- @sub_thread = Thread.new do
60
- until Thread.current[:stop] do
61
- begin
62
- @sub_buffer.dequeue_buffered_messages(self.subscribe_timeout)
63
- check_exception_and_shutdown
64
- rescue => e
65
- Thread.current[:exception] = e
66
- end
67
- check_exception_and_shutdown
68
- end
69
- end
70
- end
71
- end
72
- end
73
- end
@@ -1,52 +0,0 @@
1
- module Euston
2
- module MessageBufferDaemon
3
- class EventStoreComponent
4
- include Euston::Daemons::ComponentShutdown
5
-
6
- attr_reader :cli_thread
7
- attr_writer :wait_time
8
-
9
- def initialize channel
10
- @channel = channel
11
- end
12
-
13
- def wait_time
14
- @wait_time ||= 0.2
15
- end
16
-
17
- def start
18
- @cli_thread = Thread.new do
19
- @event_buffer = Euston::RabbitMq::ReadModel::MessageBuffer.events
20
- @event_store = Euston::RabbitMq.event_store
21
-
22
- until Thread.current[:stop] do
23
- begin
24
- loop do
25
- commits = @event_store.get_undispatched_commits
26
- break if commits.empty?
27
- commits.each do |commit|
28
- commit.events.each { |event| @event_buffer.buffer_new_message event.to_hash }
29
- @event_store.mark_commit_as_dispatched commit
30
- end
31
- end
32
- rescue => e
33
- Thread.current[:exception] = e
34
- end
35
- check_exception_and_shutdown
36
- sleep self.wait_time
37
- end
38
- end
39
- end
40
-
41
- def thread_state
42
- @cli_thread.status
43
- end
44
-
45
- def stop
46
- @cli_thread[:stop] = true
47
- @channel.disconnect
48
- end
49
-
50
- end
51
- end
52
- end
@@ -1,54 +0,0 @@
1
- module Euston
2
- class MessageLogger
3
- class << self
4
- def commands_logger channel
5
- self.new channel, :commands
6
- end
7
-
8
- def events_logger channel
9
- self.new channel, :events
10
- end
11
- end
12
-
13
- include Euston::RabbitMq::Exchanges
14
- include Euston::RabbitMq::Queues
15
-
16
- def initialize channel, exchange_name
17
- @channel = channel
18
- @exchange = get_exchange channel, exchange_name
19
- @read_model = Euston::MessageBufferDaemon::ReadModel::MessageLog.send exchange_name
20
-
21
- @queue = get_queue channel, "#{exchange_name}_log"
22
- @queue.bind @exchange, :routing_key => "#{exchange_name}.#"
23
-
24
- @queue.when(:message_decode_failed => method(:log_failure),
25
- :message_failed => method(:message_failed),
26
- :message_received => method(:write_message_to_log))
27
- end
28
-
29
- def message_failed(message, error, header)
30
- log_failure message, error
31
- header.ack
32
- end
33
-
34
- def disconnect
35
- @channel.disconnect
36
- end
37
-
38
- def start
39
- @queue.safe_subscribe
40
- end
41
-
42
- def log_failure message, error
43
- text = "A log queue subscription failed. [Error] #{error.message} [Payload] #{message}"
44
- err = Euston::RabbitMq::MessageDecodeFailedError.new text
45
- err.set_backtrace error.backtrace
46
-
47
- Safely.report! err
48
- end
49
-
50
- def write_message_to_log message
51
- @read_model.log_new_message message
52
- end
53
- end
54
- end
@@ -1,56 +0,0 @@
1
- module Euston
2
- module MessageBufferDaemon
3
- class Publisher
4
- class << self
5
- def commands_buffer channel
6
- self.new channel, :commands
7
- end
8
-
9
- def events_buffer channel
10
- self.new channel, :events
11
- end
12
- end
13
-
14
- include Euston::RabbitMq::Exchanges
15
-
16
- def initialize channel, exchange_name
17
- @read_model = Euston::RabbitMq::ReadModel::MessageBuffer.send exchange_name
18
-
19
- @channel = channel
20
- @exchange = get_exchange @channel, exchange_name
21
- end
22
-
23
- def disconnect
24
- @channel.disconnect
25
- end
26
-
27
- def dispatch_one_due_message
28
- if (message = @read_model.find_next_message)
29
- @read_model.set_next_attempt message
30
- protected_publish(message)
31
- end
32
- end
33
-
34
- def dispatch_due_messages
35
- dispatched = 0
36
- @read_model.find_due_messages.each do |message|
37
- @read_model.set_next_attempt message
38
- break unless protected_publish(message)
39
- dispatched += 1
40
- end
41
- dispatched
42
- end
43
-
44
- def protected_publish(message)
45
- ret = true
46
- begin
47
- @exchange.publish message['json'], default_publish_options.merge(:routing_key => "#{@exchange.name}.#{message['type']}")
48
- rescue NativeException => e
49
- Thread.current[:exception] = e
50
- ret = false
51
- end
52
- ret
53
- end
54
- end
55
- end
56
- end
@@ -1,14 +0,0 @@
1
- module Euston
2
- module MessageBufferDaemon
3
- module Settings
4
- def self.configure(cfg=nil)
5
- @config ||= {}
6
- @config.merge!(cfg) if cfg && cfg.is_a?(Hash)
7
- end
8
-
9
- def self.debug
10
- @config[:debug]
11
- end
12
- end
13
- end
14
- end
@@ -1,60 +0,0 @@
1
- module Euston
2
- module MessageBufferDaemon
3
- class Subscriber
4
- class << self
5
- def commands_buffer channel
6
- self.new channel, :commands
7
- end
8
-
9
- def events_buffer channel
10
- self.new channel, :events
11
- end
12
- end
13
-
14
- include Euston::RabbitMq::Exchanges
15
- include Euston::RabbitMq::Queues
16
- include Hollywood
17
-
18
- def initialize channel, exchange_name
19
- @read_model = Euston::RabbitMq::ReadModel::MessageBuffer.send exchange_name
20
-
21
- @channel = channel
22
- @channel.prefetch(1)
23
- @exchange = get_exchange @channel, exchange_name
24
- @queue = get_queue @channel, "#{exchange_name}_buffer"
25
- @queue.bind @exchange, :routing_key => "#{exchange_name}.#"
26
-
27
- @queue.when(:message_decode_failed => method(:log_failure),
28
- :message_failed => method(:message_failed),
29
- :message_received => method(:remove_message_from_buffer))
30
-
31
- @consumer = @queue.consumer
32
- end
33
-
34
- def disconnect
35
- @channel.disconnect
36
- end
37
-
38
- def dequeue_buffered_messages(timeout) #will block until message or timeout
39
- @queue.safe_subscribe_with_timeout(@consumer, timeout)
40
- end
41
-
42
- def log_failure message, error
43
- text = "A buffer queue subscription failed. [Error] #{error.message} [Payload] #{message}"
44
- err = Euston::RabbitMq::MessageDecodeFailedError.new text
45
- err.set_backtrace error.backtrace
46
-
47
- Safely.report! err
48
- end
49
-
50
- def message_failed(message, error, header)
51
- header.ack!
52
- log_failure message, error
53
- end
54
-
55
- def remove_message_from_buffer message
56
- @read_model.remove_published_message message[:headers][:id]
57
- end
58
- end
59
- end
60
- end